c# 文本编码识别

  1 public class TextEncodingDetect
  2     {
  3         #region Fields
  4 
  5         private readonly byte[] utf16LEBOM = { 0xFF, 0xFE };
  6         private readonly byte[] utf16BEBOM = { 0xFE, 0xFF };
  7         private readonly byte[] utf8BOM = { 0xEF, 0xBB, 0xBF };
  8 
  9         private bool nullSuggestsBinary = true;
 10         private double utf16ExpectedNullPercent = 70;
 11         private double utf16UnexpectedNullPercent = 10;
 12 
 13         #endregion
 14 
 15         #region Enums
 16 
 17         public enum Encoding
 18         {
 19             None,               // Unknown or binary
 20             ANSI,               // 0-255
 21             ASCII,              // 0-127
 22             UTF8_BOM,           // UTF8 with BOM
 23             UTF8_NOBOM,         // UTF8 without BOM
 24             UTF16_LE_BOM,       // UTF16 LE with BOM
 25             UTF16_LE_NOBOM,     // UTF16 LE without BOM
 26             UTF16_BE_BOM,       // UTF16-BE with BOM
 27             UTF16_BE_NOBOM      // UTF16-BE without BOM
 28         }
 29 
 30         #endregion
 31 
 32         #region Properties
 33 
 34         public bool NullSuggestsBinary
 35         {
 36             set
 37             {
 38                 this.nullSuggestsBinary = value;
 39             }
 40         }
 41 
 42         public double Utf16ExpectedNullPercent
 43         {
 44             set
 45             {
 46                 if (value > 0 && value < 100)
 47                 {
 48                     this.utf16ExpectedNullPercent = value;
 49                 }
 50             }
 51         }
 52 
 53         public double Utf16UnexpectedNullPercent
 54         {
 55             set
 56             {
 57                 if (value > 0 && value < 100)
 58                 {
 59                     this.utf16UnexpectedNullPercent = value;
 60                 }
 61             }
 62         }
 63 
 64         #endregion
 65 
 66         public static int GetBOMLengthFromEncodingMode(Encoding encoding)
 67         {
 68             int length = 0;
 69 
 70             if (encoding == Encoding.UTF16_BE_BOM || encoding == Encoding.UTF16_LE_BOM)
 71             {
 72                 length = 2;
 73             }
 74             else if (encoding == Encoding.UTF8_BOM)
 75             {
 76                 length = 3;
 77             }
 78 
 79             return length;
 80         }
 81 
 82    
 83         /// <summary>
 84         /// 85         /// </summary>
 86         /// <param name="buffer"></param>
 87         /// <param name="size"></param>
 88         /// <returns></returns>
 89         public Encoding DetectEncoding(byte[] buffer, int size)
 90         {
 91             // First check if we have a BOM and return that if so
 92             Encoding encoding = this.CheckBOM(buffer, size);
 93             if (encoding != Encoding.None)
 94             {
 95                 return encoding;
 96             }
 97 
 98             // Now check for valid UTF8
 99             encoding = this.CheckUTF8(buffer, size);
100             if (encoding != Encoding.None)
101             {
102                 return encoding;
103             }
104 
105             // Now try UTF16 
106             encoding = this.CheckUTF16NewlineChars(buffer, size);
107             if (encoding != Encoding.None)
108             {
109                 return encoding;
110             }
111 
112             encoding = this.CheckUTF16ASCII(buffer, size);
113             if (encoding != Encoding.None)
114             {
115                 return encoding;
116             }
117 
118             // ANSI or None (binary) then
119             if (!this.DoesContainNulls(buffer, size))
120             {
121                 return Encoding.ANSI;
122             }
123             else
124             {
125                 // Found a null, return based on the preference in null_suggests_binary_
126                 if (this.nullSuggestsBinary)
127                 {
128                     return Encoding.None;
129                 }
130                 else
131                 {
132                     return Encoding.ANSI;
133                 }
134             }
135         }
136 
137 
138         public Encoding CheckBOM(byte[] buffer, int size)
139         {
140             // Check for BOM
141             if (size >= 2 && buffer[0] == this.utf16LEBOM[0] && buffer[1] == this.utf16LEBOM[1])
142             {
143                 return Encoding.UTF16_LE_BOM;
144             }
145             else if (size >= 2 && buffer[0] == this.utf16BEBOM[0] && buffer[1] == this.utf16BEBOM[1])
146             {
147                 return Encoding.UTF16_BE_BOM;
148             }
149             else if (size >= 3 && buffer[0] == this.utf8BOM[0] && buffer[1] == this.utf8BOM[1] && buffer[2] == this.utf8BOM[2])
150             {
151                 return Encoding.UTF8_BOM;
152             }
153             else
154             {
155                 return Encoding.None;
156             }
157         }
158 
159         ///////////////////////////////////////////////////////////////////////////////
160         // Checks if a buffer contains valid utf8. Returns:
161         // None - not valid utf8
162         // UTF8_NOBOM - valid utf8 encodings and multibyte sequences
163         // ASCII - Only data in the 0-127 range. 
164         ///////////////////////////////////////////////////////////////////////////////
165 
166         private Encoding CheckUTF8(byte[] buffer, int size)
167         {
168             // UTF8 Valid sequences
169             // 0xxxxxxx  ASCII
170             // 110xxxxx 10xxxxxx  2-byte
171             // 1110xxxx 10xxxxxx 10xxxxxx  3-byte
172             // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx  4-byte
173             //
174             // Width in UTF8
175             // Decimal      Width
176             // 0-127        1 byte
177             // 194-223      2 bytes
178             // 224-239      3 bytes
179             // 240-244      4 bytes
180             //
181             // Subsequent chars are in the range 128-191
182             bool only_saw_ascii_range = true;
183             uint pos = 0;
184             int more_chars;
185 
186             while (pos < size)
187             {
188                 byte ch = buffer[pos++];
189 
190                 if (ch == 0 && this.nullSuggestsBinary)
191                 {
192                     return Encoding.None;
193                 }
194                 else if (ch <= 127)
195                 {
196                     // 1 byte
197                     more_chars = 0;
198                 }
199                 else if (ch >= 194 && ch <= 223)
200                 {
201                     // 2 Byte
202                     more_chars = 1;
203                 }
204                 else if (ch >= 224 && ch <= 239)
205                 {
206                     // 3 Byte
207                     more_chars = 2;
208                 }
209                 else if (ch >= 240 && ch <= 244)
210                 {
211                     // 4 Byte
212                     more_chars = 3;
213                 }
214                 else
215                 {
216                     return Encoding.None;               // Not utf8
217                 }
218 
219                 // Check secondary chars are in range if we are expecting any
220                 while (more_chars > 0 && pos < size)
221                 {
222                     only_saw_ascii_range = false;       // Seen non-ascii chars now
223 
224                     ch = buffer[pos++];
225                     if (ch < 128 || ch > 191)
226                     {
227                         return Encoding.None;           // Not utf8
228                     }
229 
230                     --more_chars;
231                 }
232             }
233 
234             // If we get to here then only valid UTF-8 sequences have been processed
235 
236             // If we only saw chars in the range 0-127 then we can‘t assume UTF8 (the caller will need to decide)
237             if (only_saw_ascii_range)
238             {
239                 return Encoding.ASCII;
240             }
241             else
242             {
243                 return Encoding.UTF8_NOBOM;
244             }
245         }
246 
247         ///////////////////////////////////////////////////////////////////////////////
248         // Checks if a buffer contains text that looks like utf16 by scanning for 
249         // newline chars that would be present even in non-english text.
250         // Returns:
251         // None - not valid utf16
252         // UTF16_LE_NOBOM - looks like utf16 le
253         // UTF16_BE_NOBOM - looks like utf16 be
254         ///////////////////////////////////////////////////////////////////////////////
255 
256         private Encoding CheckUTF16NewlineChars(byte[] buffer, int size)
257         {
258             if (size < 2)
259             {
260                 return Encoding.None;
261             }
262 
263             // Reduce size by 1 so we don‘t need to worry about bounds checking for pairs of bytes
264             size--;
265 
266             int le_control_chars = 0;
267             int be_control_chars = 0;
268             byte ch1, ch2;
269 
270             uint pos = 0;
271             while (pos < size)
272             {
273                 ch1 = buffer[pos++];
274                 ch2 = buffer[pos++];
275 
276                 if (ch1 == 0)
277                 {
278                     if (ch2 == 0x0a || ch2 == 0x0d)
279                     {
280                         ++be_control_chars;
281                     }
282                 }
283                 else if (ch2 == 0)
284                 {
285                     if (ch1 == 0x0a || ch1 == 0x0d)
286                     {
287                         ++le_control_chars;
288                     }
289                 }
290 
291                 // If we are getting both LE and BE control chars then this file is not utf16
292                 if (le_control_chars > 0 && be_control_chars > 0)
293                 {
294                     return Encoding.None;
295                 }
296             }
297 
298             if (le_control_chars > 0)
299             {
300                 return Encoding.UTF16_LE_NOBOM;
301             }
302             else if (be_control_chars > 0)
303             {
304                 return Encoding.UTF16_BE_NOBOM;
305             }
306             else
307             {
308                 return Encoding.None;
309             }
310         }
311 
312         ///////////////////////////////////////////////////////////////////////////////
313         // Checks if a buffer contains text that looks like utf16. This is done based
314         // the use of nulls which in ASCII/script like text can be useful to identify.
315         // Returns:
316         // None - not valid utf16
317         // UTF16_LE_NOBOM - looks like utf16 le
318         // UTF16_BE_NOBOM - looks like utf16 be
319         ///////////////////////////////////////////////////////////////////////////////
320 
321         private Encoding CheckUTF16ASCII(byte[] buffer, int size)
322         {
323             int num_odd_nulls = 0;
324             int num_even_nulls = 0;
325 
326             // Get even nulls
327             uint pos = 0;
328             while (pos < size)
329             {
330                 if (buffer[pos] == 0)
331                 {
332                     num_even_nulls++;
333                 }
334 
335                 pos += 2;
336             }
337 
338             // Get odd nulls
339             pos = 1;
340             while (pos < size)
341             {
342                 if (buffer[pos] == 0)
343                 {
344                     num_odd_nulls++;
345                 }
346 
347                 pos += 2;
348             }
349 
350             double even_null_threshold = (num_even_nulls * 2.0) / size;
351             double odd_null_threshold = (num_odd_nulls * 2.0) / size;
352             double expected_null_threshold = this.utf16ExpectedNullPercent / 100.0;
353             double unexpected_null_threshold = this.utf16UnexpectedNullPercent / 100.0;
354 
355             // Lots of odd nulls, low number of even nulls
356             if (even_null_threshold < unexpected_null_threshold && odd_null_threshold > expected_null_threshold)
357             {
358                 return Encoding.UTF16_LE_NOBOM;
359             }
360 
361             // Lots of even nulls, low number of odd nulls
362             if (odd_null_threshold < unexpected_null_threshold && even_null_threshold > expected_null_threshold)
363             {
364                 return Encoding.UTF16_BE_NOBOM;
365             }
366 
367             // Don‘t know
368             return Encoding.None;
369         }
370 
371         ///////////////////////////////////////////////////////////////////////////////
372         // Checks if a buffer contains any nulls. Used to check for binary vs text data.
373         ///////////////////////////////////////////////////////////////////////////////
374 
375         private bool DoesContainNulls(byte[] buffer, int size)
376         {
377             uint pos = 0;
378             while (pos < size)
379             {
380                 if (buffer[pos++] == 0)
381                 {
382                     return true;
383                 }
384             }
385 
386             return false;
387         }
388     }