1 /*
2 **********************************************************************
3 * Copyright (C) 2005-2012, International Business Machines
4 * Corporation and others. All Rights Reserved.
5 **********************************************************************
6 */
7
8 #include "unicode/utypes.h"
9
10 #if !UCONFIG_NO_CONVERSION
11
12 #include "csrucode.h"
13 #include "csmatch.h"
14
15 U_NAMESPACE_BEGIN
16
~CharsetRecog_Unicode()17 CharsetRecog_Unicode::~CharsetRecog_Unicode()
18 {
19 // nothing to do
20 }
21
~CharsetRecog_UTF_16_BE()22 CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
23 {
24 // nothing to do
25 }
26
getName() const27 const char *CharsetRecog_UTF_16_BE::getName() const
28 {
29 return "UTF-16BE";
30 }
31
match(InputText * textIn,CharsetMatch * results) const32 UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
33 {
34 const uint8_t *input = textIn->fRawInput;
35 int32_t confidence = 0;
36
37 if (input[0] == 0xFE && input[1] == 0xFF) {
38 confidence = 100;
39 }
40
41 // TODO: Do some statastics to check for unsigned UTF-16BE
42 results->set(textIn, this, confidence);
43 return (confidence > 0);
44 }
45
~CharsetRecog_UTF_16_LE()46 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
47 {
48 // nothing to do
49 }
50
getName() const51 const char *CharsetRecog_UTF_16_LE::getName() const
52 {
53 return "UTF-16LE";
54 }
55
match(InputText * textIn,CharsetMatch * results) const56 UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
57 {
58 const uint8_t *input = textIn->fRawInput;
59 int32_t confidence = 0;
60
61 if (input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) {
62 confidence = 100;
63 }
64
65 // TODO: Do some statastics to check for unsigned UTF-16LE
66 results->set(textIn, this, confidence);
67 return (confidence > 0);
68 }
69
~CharsetRecog_UTF_32()70 CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
71 {
72 // nothing to do
73 }
74
match(InputText * textIn,CharsetMatch * results) const75 UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
76 {
77 const uint8_t *input = textIn->fRawInput;
78 int32_t limit = (textIn->fRawLength / 4) * 4;
79 int32_t numValid = 0;
80 int32_t numInvalid = 0;
81 bool hasBOM = FALSE;
82 int32_t confidence = 0;
83
84 if (getChar(input, 0) == 0x0000FEFFUL) {
85 hasBOM = TRUE;
86 }
87
88 for(int32_t i = 0; i < limit; i += 4) {
89 int32_t ch = getChar(input, i);
90
91 if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
92 numInvalid += 1;
93 } else {
94 numValid += 1;
95 }
96 }
97
98
99 // Cook up some sort of confidence score, based on presense of a BOM
100 // and the existence of valid and/or invalid multi-byte sequences.
101 if (hasBOM && numInvalid==0) {
102 confidence = 100;
103 } else if (hasBOM && numValid > numInvalid*10) {
104 confidence = 80;
105 } else if (numValid > 3 && numInvalid == 0) {
106 confidence = 100;
107 } else if (numValid > 0 && numInvalid == 0) {
108 confidence = 80;
109 } else if (numValid > numInvalid*10) {
110 // Probably corruput UTF-32BE data. Valid sequences aren't likely by chance.
111 confidence = 25;
112 }
113
114 results->set(textIn, this, confidence);
115 return (confidence > 0);
116 }
117
~CharsetRecog_UTF_32_BE()118 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
119 {
120 // nothing to do
121 }
122
getName() const123 const char *CharsetRecog_UTF_32_BE::getName() const
124 {
125 return "UTF-32BE";
126 }
127
getChar(const uint8_t * input,int32_t index) const128 int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
129 {
130 return input[index + 0] << 24 | input[index + 1] << 16 |
131 input[index + 2] << 8 | input[index + 3];
132 }
133
~CharsetRecog_UTF_32_LE()134 CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
135 {
136 // nothing to do
137 }
138
getName() const139 const char *CharsetRecog_UTF_32_LE::getName() const
140 {
141 return "UTF-32LE";
142 }
143
getChar(const uint8_t * input,int32_t index) const144 int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
145 {
146 return input[index + 3] << 24 | input[index + 2] << 16 |
147 input[index + 1] << 8 | input[index + 0];
148 }
149
150 U_NAMESPACE_END
151 #endif
152
153