Find Usages Diff Raw Download HTML Widget
Start line:  
End line:  

Snippet Preview

Snippet HTML Code

Stack Overflow Questions 
1
  /*
2
   * Copyright 2003-2007 the original author or authors.
3
   *
4
   * Licensed under the Apache License, Version 2.0 (the "License");
5
   * you may not use this file except in compliance with the License.
6
   * You may obtain a copy of the License at
7
   *
8
   *     http://www.apache.org/licenses/LICENSE-2.0
9
   *
10
  * Unless required by applicable law or agreed to in writing, software
11
  * distributed under the License is distributed on an "AS IS" BASIS,
12
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
  * See the License for the specific language governing permissions and
14
  * limitations under the License.
15
  */
16
 
17
 package groovy.util;
18
 
19
 import java.io.*;

Utility class to guess the encoding of a given text file.

Unicode files encoded in UTF-16 (low or big endian) or UTF-8 files with a Byte Order Marker are correctly discovered. For UTF-8 files with no BOM, if the buffer is wide enough, the charset should also be discovered.

A byte buffer of 4KB is used to be able to guess the encoding.

Usage:

 CharsetToolkit toolkit = new CharsetToolkit(file);

 // guess the encoding
 Charset guessedCharset = toolkit.getCharset();

 // create a reader with the correct charset
 BufferedReader reader = toolkit.getReader();

 // read the file content
 String line;
 while ((line = br.readLine())!= null)
 {
     System.out.println(line);
 }

Author(s):
Guillaume Laforge
51
 
52
 public class CharsetToolkit {
53
     private byte[] buffer;
54
     private Charset defaultCharset;
55
     private Charset charset;
56
     private boolean enforce8Bit = true;
57
     private final File file;
58
     private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];

    
Constructor of the CharsetToolkit utility class.

Parameters:
file of which we want to know the encoding.
64
 
65
     public CharsetToolkit(File file) throws IOException {
66
         this. = file;
67
         this. = getDefaultSystemCharset();
68
         this. = null;
69
         InputStream input = new FileInputStream(file);
70
         try {
71
             byte[] bytes = new byte[4096];
72
             int bytesRead = input.read(bytes);
73
             if (bytesRead == -1) {
74
                 this. = ;
75
             }
76
             else if (bytesRead < 4096) {
77
                 byte[] bytesToGuess = new byte[bytesRead];
78
                 System.arraycopy(bytes, 0, bytesToGuess, 0, bytesRead);
79
                 this. = bytesToGuess;
80
             }
81
             else {
82
                 this. = bytes;
83
             }
84
         } finally {
85
             try {input.close();} catch (IOException e){
86
                 // IGNORE
87
             }
88
         }
89
     }

    
Defines the default Charset used in case the buffer represents an 8-bit Charset.

Parameters:
defaultCharset the default Charset to be returned if an 8-bit Charset is encountered.
97
 
98
     public void setDefaultCharset(Charset defaultCharset) {
99
         if (defaultCharset != null)
100
            this. = defaultCharset;
101
        else
102
            this. = getDefaultSystemCharset();
103
    }
105
    public Charset getCharset() {
106
        if (this. == null)
107
            this. = guessEncoding();
108
        return ;
109
    }

    
If US-ASCII is recognized, enforce to return the default encoding, rather than US-ASCII. It might be a file without any special character in the range 128-255, but that may be or become a file encoded with the default charset rather than US-ASCII.

Parameters:
enforce a boolean specifying the use or not of US-ASCII.
118
    public void setEnforce8Bit(boolean enforce) {
119
        this. = enforce;
120
    }

    
Gets the enforce8Bit flag, in case we do not want to ever get a US-ASCII encoding.

Returns:
a boolean representing the flag of use of US-ASCII.
127
    public boolean getEnforce8Bit() {
128
        return this.;
129
    }

    
Retrieves the default Charset 
134
    public Charset getDefaultCharset() {
135
        return ;
136
    }

    

Guess the encoding of the provided buffer.

If Byte Order Markers are encountered at the beginning of the buffer, we immediately return the charset implied by this BOM. Otherwise, the file would not be a human readable text file.

If there is no BOM, this method tries to discern whether the file is UTF-8 or not. If it is not UTF-8, we assume the encoding is the default system encoding (of course, it might be any 8-bit charset, but usually, an 8-bit charset is the default one).

It is possible to discern UTF-8 thanks to the pattern of characters with a multi-byte sequence.

 UCS-4 range (hex.)        UTF-8 octet sequence (binary)
 0000 0000-0000 007F       0xxxxxxx
 0000 0080-0000 07FF       110xxxxx 10xxxxxx
 0000 0800-0000 FFFF       1110xxxx 10xxxxxx 10xxxxxx
 0001 0000-001F FFFF       11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
 0020 0000-03FF FFFF       111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
 0400 0000-7FFF FFFF       1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

With UTF-8, 0xFE and 0xFF never appear.

Returns:
the Charset recognized.
162
    private Charset guessEncoding() {
163
        // if the file has a Byte Order Marker, we can assume the file is in UTF-xx
164
        // otherwise, the file would not be human readable
165
        if (hasUTF8Bom())
166
            return Charset.forName("UTF-8");
167
        if (hasUTF16LEBom())
168
            return Charset.forName("UTF-16LE");
169
        if (hasUTF16BEBom())
170
            return Charset.forName("UTF-16BE");
172
        // if a byte has its most significant bit set, the file is in UTF-8 or in the default encoding
173
        // otherwise, the file is in US-ASCII
174
        boolean highOrderBit = false;
176
        // if the file is in UTF-8, high order bytes must have a certain value, in order to be valid
177
        // if it's not the case, we can assume the encoding is the default encoding of the system
178
        boolean validU8Char = true;
180
        // TODO the buffer is not read up to the end, but up to length - 6
182
        int length = .;
183
        int i = 0;
184
        while (i < length - 6) {
185
            byte b0 = [i];
186
            byte b1 = [i + 1];
187
            byte b2 = [i + 2];
188
            byte b3 = [i + 3];
189
            byte b4 = [i + 4];
190
            byte b5 = [i + 5];
191
            if (b0 < 0) {
192
                // a high order bit was encountered, thus the encoding is not US-ASCII
193
                // it may be either an 8-bit encoding or UTF-8
194
                highOrderBit = true;
195
                // a two-bytes sequence was encountered
196
                if (isTwoBytesSequence(b0)) {
197
                    // there must be one continuation byte of the form 10xxxxxx,
198
                    // otherwise the following characteris is not a valid UTF-8 construct
199
                    if (!isContinuationChar(b1))
200
                        validU8Char = false;
201
                    else
202
                        i++;
203
                }
204
                // a three-bytes sequence was encountered
205
                else if (isThreeBytesSequence(b0)) {
206
                    // there must be two continuation bytes of the form 10xxxxxx,
207
                    // otherwise the following characteris is not a valid UTF-8 construct
208
                    if (!(isContinuationChar(b1) && isContinuationChar(b2)))
209
                        validU8Char = false;
210
                    else
211
                        i += 2;
212
                }
213
                // a four-bytes sequence was encountered
214
                else if (isFourBytesSequence(b0)) {
215
                    // there must be three continuation bytes of the form 10xxxxxx,
216
                    // otherwise the following characteris is not a valid UTF-8 construct
217
                    if (!(isContinuationChar(b1) && isContinuationChar(b2) && isContinuationChar(b3)))
218
                        validU8Char = false;
219
                    else
220
                        i += 3;
221
                }
222
                // a five-bytes sequence was encountered
223
                else if (isFiveBytesSequence(b0)) {
224
                    // there must be four continuation bytes of the form 10xxxxxx,
225
                    // otherwise the following characteris is not a valid UTF-8 construct
226
                    if (!(isContinuationChar(b1)
227
                        && isContinuationChar(b2)
228
                        && isContinuationChar(b3)
229
                        && isContinuationChar(b4)))
230
                        validU8Char = false;
231
                    else
232
                        i += 4;
233
                }
234
                // a six-bytes sequence was encountered
235
                else if (isSixBytesSequence(b0)) {
236
                    // there must be five continuation bytes of the form 10xxxxxx,
237
                    // otherwise the following characteris is not a valid UTF-8 construct
238
                    if (!(isContinuationChar(b1)
239
                        && isContinuationChar(b2)
240
                        && isContinuationChar(b3)
241
                        && isContinuationChar(b4)
242
                        && isContinuationChar(b5)))
243
                        validU8Char = false;
244
                    else
245
                        i += 5;
246
                }
247
                else
248
                    validU8Char = false;
249
            }
250
            if (!validU8Char)
251
                break;
252
            i++;
253
        }
254
        // if no byte with an high order bit set, the encoding is US-ASCII
255
        // (it might have been UTF-7, but this encoding is usually internally used only by mail systems)
256
        if (!highOrderBit) {
257
            // returns the default charset rather than US-ASCII if the enforce8Bit flag is set.
258
            if (this.)
259
                return this.;
260
            else
261
                return Charset.forName("US-ASCII");
262
        }
263
        // if no invalid UTF-8 were encountered, we can assume the encoding is UTF-8,
264
        // otherwise the file would not be human readable
265
        if (validU8Char)
266
            return Charset.forName("UTF-8");
267
        // finally, if it's not UTF-8 nor US-ASCII, let's assume the encoding is the default encoding
268
        return this.;
269
    }

    
If the byte has the form 10xxxxx, then it's a continuation byte of a multiple byte character;

Parameters:
b a byte.
Returns:
true if it's a continuation char.
277
    private static boolean isContinuationChar(byte b) {
278
        return -128 <= b && b <= -65;
279
    }

    
If the byte has the form 110xxxx, then it's the first byte of a two-bytes sequence character.

Parameters:
b a byte.
Returns:
true if it's the first byte of a two-bytes sequence.
287
    private static boolean isTwoBytesSequence(byte b) {
288
        return -64 <= b && b <= -33;
289
    }

    
If the byte has the form 1110xxx, then it's the first byte of a three-bytes sequence character.

Parameters:
b a byte.
Returns:
true if it's the first byte of a three-bytes sequence.
297
    private static boolean isThreeBytesSequence(byte b) {
298
        return -32 <= b && b <= -17;
299
    }

    
If the byte has the form 11110xx, then it's the first byte of a four-bytes sequence character.

Parameters:
b a byte.
Returns:
true if it's the first byte of a four-bytes sequence.
307
    private static boolean isFourBytesSequence(byte b) {
308
        return -16 <= b && b <= -9;
309
    }

    
If the byte has the form 11110xx, then it's the first byte of a five-bytes sequence character.

Parameters:
b a byte.
Returns:
true if it's the first byte of a five-bytes sequence.
317
    private static boolean isFiveBytesSequence(byte b) {
318
        return -8 <= b && b <= -5;
319
    }

    
If the byte has the form 1110xxx, then it's the first byte of a six-bytes sequence character.

Parameters:
b a byte.
Returns:
true if it's the first byte of a six-bytes sequence.
327
    private static boolean isSixBytesSequence(byte b) {
328
        return -4 <= b && b <= -3;
329
    }

    
Retrieve the default charset of the system.

Returns:
the default Charset.
336
    public static Charset getDefaultSystemCharset() {
337
        return Charset.forName(System.getProperty("file.encoding"));
338
    }

    
Has a Byte Order Marker for UTF-8 (Used by Microsoft's Notepad and other editors).

Returns:
true if the buffer has a BOM for UTF8.
345
    public boolean hasUTF8Bom() {
346
        if (. >= 3)
347
            return ([0] == -17 && [1] == -69 && [2] == -65);
348
        else
349
            return false;
350
    }

    
Has a Byte Order Marker for UTF-16 Low Endian (ucs-2le, ucs-4le, and ucs-16le).

Returns:
true if the buffer has a BOM for UTF-16 Low Endian.
358
    public boolean hasUTF16LEBom() {
359
        if (. >= 2)
360
            return ([0] == -1 && [1] == -2);
361
        else
362
            return false;
363
    }

    
Has a Byte Order Marker for UTF-16 Big Endian (utf-16 and ucs-2).

Returns:
true if the buffer has a BOM for UTF-16 Big Endian.
371
    public boolean hasUTF16BEBom() {
372
        if (. >= 2)
373
            return ([0] == -2 && [1] == -1);
374
        else
375
            return false;
376
    }

    
Gets a BufferedReader (indeed a LineNumberReader) from the File specified in the constructor of CharsetToolkit using the charset discovered or the default charset if an 8-bit Charset is encountered.

Returns:
a BufferedReader
Throws:
java.io.FileNotFoundException if the file is not found.
386
    public BufferedReader getReader() throws FileNotFoundException {
387
        LineNumberReader reader = new LineNumberReader(new InputStreamReader(new FileInputStream(), getCharset()));
388
        if (hasUTF8Bom() || hasUTF16LEBom() || hasUTF16BEBom()) {
389
            try {
390
                reader.read();
391
            }
392
            catch (IOException e) {
393
                // should never happen, as a file with no content
394
                // but with a BOM has at least one char
395
            }
396
        }
397
        return reader;
398
    }

    
Retrieves all the available Charsets on the platform, among which the default charset.

Returns:
an array of Charsets.
406
    public static Charset[] getAvailableCharsets() {
407
        Collection collection = Charset.availableCharsets().values();
408
        return (Charset[]) collection.toArray(new Charset[collection.size()]);
409
    }
New to GrepCode? Check out our FAQ X