1 /*
2 * Copyright (C) 2013, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "suggest/policyimpl/dictionary/header/header_read_write_utils.h"
18
19 #include <cctype>
20 #include <cstdio>
21 #include <vector>
22
23 #include "defines.h"
24 #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"
25 #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"
26
27 namespace latinime {
28
29 const int HeaderReadWriteUtils::MAX_ATTRIBUTE_KEY_LENGTH = 256;
30 const int HeaderReadWriteUtils::MAX_ATTRIBUTE_VALUE_LENGTH = 256;
31
32 const int HeaderReadWriteUtils::HEADER_MAGIC_NUMBER_SIZE = 4;
33 const int HeaderReadWriteUtils::HEADER_DICTIONARY_VERSION_SIZE = 2;
34 const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2;
35 const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4;
36
37 const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0;
38 // Flags for special processing
39 // Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or
40 // something very bad (like, the apocalypse) will happen. Please update both at the same time.
41 const HeaderReadWriteUtils::DictionaryFlags
42 HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1;
43 const HeaderReadWriteUtils::DictionaryFlags
44 HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2;
45 const HeaderReadWriteUtils::DictionaryFlags
46 HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4;
47
48 // Note that these are corresponding definitions in Java side in FormatSpec.FileHeader.
49 const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE";
50 const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
51 "REQUIRES_GERMAN_UMLAUT_PROCESSING";
52 const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY =
53 "REQUIRES_FRENCH_LIGATURE_PROCESSING";
54
getHeaderSize(const uint8_t * const dictBuf)55 /* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) {
56 // See the format of the header in the comment in
57 // BinaryDictionaryFormatUtils::detectFormatVersion()
58 return ByteArrayUtils::readUint32(dictBuf, HEADER_MAGIC_NUMBER_SIZE
59 + HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE);
60 }
61
62 /* static */ HeaderReadWriteUtils::DictionaryFlags
getFlags(const uint8_t * const dictBuf)63 HeaderReadWriteUtils::getFlags(const uint8_t *const dictBuf) {
64 return ByteArrayUtils::readUint16(dictBuf,
65 HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE);
66 }
67
68 /* static */ HeaderReadWriteUtils::DictionaryFlags
createAndGetDictionaryFlagsUsingAttributeMap(const HeaderReadWriteUtils::AttributeMap * const attributeMap)69 HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap(
70 const HeaderReadWriteUtils::AttributeMap *const attributeMap) {
71 const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap,
72 REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */);
73 const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap,
74 REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */);
75 const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap,
76 SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */);
77 DictionaryFlags dictflags = NO_FLAGS;
78 dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0;
79 dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0;
80 dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0;
81 return dictflags;
82 }
83
fetchAllHeaderAttributes(const uint8_t * const dictBuf,AttributeMap * const headerAttributes)84 /* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf,
85 AttributeMap *const headerAttributes) {
86 const int headerSize = getHeaderSize(dictBuf);
87 int pos = getHeaderOptionsPosition();
88 if (pos == NOT_A_DICT_POS) {
89 // The header doesn't have header options.
90 return;
91 }
92 int keyBuffer[MAX_ATTRIBUTE_KEY_LENGTH];
93 int valueBuffer[MAX_ATTRIBUTE_VALUE_LENGTH];
94 while (pos < headerSize) {
95 const int keyLength = ByteArrayUtils::readStringAndAdvancePosition(dictBuf,
96 MAX_ATTRIBUTE_KEY_LENGTH, keyBuffer, &pos);
97 std::vector<int> key;
98 key.insert(key.end(), keyBuffer, keyBuffer + keyLength);
99 const int valueLength = ByteArrayUtils::readStringAndAdvancePosition(dictBuf,
100 MAX_ATTRIBUTE_VALUE_LENGTH, valueBuffer, &pos);
101 std::vector<int> value;
102 value.insert(value.end(), valueBuffer, valueBuffer + valueLength);
103 headerAttributes->insert(AttributeMap::value_type(key, value));
104 }
105 }
106
writeDictionaryVersion(BufferWithExtendableBuffer * const buffer,const FormatUtils::FORMAT_VERSION version,int * const writingPos)107 /* static */ bool HeaderReadWriteUtils::writeDictionaryVersion(
108 BufferWithExtendableBuffer *const buffer, const FormatUtils::FORMAT_VERSION version,
109 int *const writingPos) {
110 if (!buffer->writeUintAndAdvancePosition(FormatUtils::MAGIC_NUMBER, HEADER_MAGIC_NUMBER_SIZE,
111 writingPos)) {
112 return false;
113 }
114 switch (version) {
115 case FormatUtils::VERSION_2:
116 // Version 2 dictionary writing is not supported.
117 return false;
118 case FormatUtils::VERSION_3:
119 return buffer->writeUintAndAdvancePosition(3 /* data */,
120 HEADER_DICTIONARY_VERSION_SIZE, writingPos);
121 default:
122 return false;
123 }
124 }
125
writeDictionaryFlags(BufferWithExtendableBuffer * const buffer,const DictionaryFlags flags,int * const writingPos)126 /* static */ bool HeaderReadWriteUtils::writeDictionaryFlags(
127 BufferWithExtendableBuffer *const buffer, const DictionaryFlags flags,
128 int *const writingPos) {
129 return buffer->writeUintAndAdvancePosition(flags, HEADER_FLAG_SIZE, writingPos);
130 }
131
writeDictionaryHeaderSize(BufferWithExtendableBuffer * const buffer,const int size,int * const writingPos)132 /* static */ bool HeaderReadWriteUtils::writeDictionaryHeaderSize(
133 BufferWithExtendableBuffer *const buffer, const int size, int *const writingPos) {
134 return buffer->writeUintAndAdvancePosition(size, HEADER_SIZE_FIELD_SIZE, writingPos);
135 }
136
writeHeaderAttributes(BufferWithExtendableBuffer * const buffer,const AttributeMap * const headerAttributes,int * const writingPos)137 /* static */ bool HeaderReadWriteUtils::writeHeaderAttributes(
138 BufferWithExtendableBuffer *const buffer, const AttributeMap *const headerAttributes,
139 int *const writingPos) {
140 for (AttributeMap::const_iterator it = headerAttributes->begin();
141 it != headerAttributes->end(); ++it) {
142 if (it->first.empty() || it->second.empty()) {
143 continue;
144 }
145 // Write a key.
146 if (!buffer->writeCodePointsAndAdvancePosition(&(it->first.at(0)), it->first.size(),
147 true /* writesTerminator */, writingPos)) {
148 return false;
149 }
150 // Write a value.
151 if (!buffer->writeCodePointsAndAdvancePosition(&(it->second.at(0)), it->second.size(),
152 true /* writesTerminator */, writingPos)) {
153 return false;
154 }
155 }
156 return true;
157 }
158
setBoolAttribute(AttributeMap * const headerAttributes,const char * const key,const bool value)159 /* static */ void HeaderReadWriteUtils::setBoolAttribute(AttributeMap *const headerAttributes,
160 const char *const key, const bool value) {
161 setIntAttribute(headerAttributes, key, value ? 1 : 0);
162 }
163
setIntAttribute(AttributeMap * const headerAttributes,const char * const key,const int value)164 /* static */ void HeaderReadWriteUtils::setIntAttribute(AttributeMap *const headerAttributes,
165 const char *const key, const int value) {
166 AttributeMap::key_type keyVector;
167 insertCharactersIntoVector(key, &keyVector);
168 setIntAttributeInner(headerAttributes, &keyVector, value);
169 }
170
setIntAttributeInner(AttributeMap * const headerAttributes,const AttributeMap::key_type * const key,const int value)171 /* static */ void HeaderReadWriteUtils::setIntAttributeInner(AttributeMap *const headerAttributes,
172 const AttributeMap::key_type *const key, const int value) {
173 AttributeMap::mapped_type valueVector;
174 char charBuf[LARGEST_INT_DIGIT_COUNT + 1];
175 snprintf(charBuf, LARGEST_INT_DIGIT_COUNT + 1, "%d", value);
176 insertCharactersIntoVector(charBuf, &valueVector);
177 (*headerAttributes)[*key] = valueVector;
178 }
179
readBoolAttributeValue(const AttributeMap * const headerAttributes,const char * const key,const bool defaultValue)180 /* static */ bool HeaderReadWriteUtils::readBoolAttributeValue(
181 const AttributeMap *const headerAttributes, const char *const key,
182 const bool defaultValue) {
183 const int intDefaultValue = defaultValue ? 1 : 0;
184 const int intValue = readIntAttributeValue(headerAttributes, key, intDefaultValue);
185 return intValue != 0;
186 }
187
readIntAttributeValue(const AttributeMap * const headerAttributes,const char * const key,const int defaultValue)188 /* static */ int HeaderReadWriteUtils::readIntAttributeValue(
189 const AttributeMap *const headerAttributes, const char *const key,
190 const int defaultValue) {
191 AttributeMap::key_type keyVector;
192 insertCharactersIntoVector(key, &keyVector);
193 return readIntAttributeValueInner(headerAttributes, &keyVector, defaultValue);
194 }
195
readIntAttributeValueInner(const AttributeMap * const headerAttributes,const AttributeMap::key_type * const key,const int defaultValue)196 /* static */ int HeaderReadWriteUtils::readIntAttributeValueInner(
197 const AttributeMap *const headerAttributes, const AttributeMap::key_type *const key,
198 const int defaultValue) {
199 AttributeMap::const_iterator it = headerAttributes->find(*key);
200 if (it != headerAttributes->end()) {
201 int value = 0;
202 bool isNegative = false;
203 for (size_t i = 0; i < it->second.size(); ++i) {
204 if (i == 0 && it->second.at(i) == '-') {
205 isNegative = true;
206 } else {
207 if (!isdigit(it->second.at(i))) {
208 // If not a number.
209 return defaultValue;
210 }
211 value *= 10;
212 value += it->second.at(i) - '0';
213 }
214 }
215 return isNegative ? -value : value;
216 }
217 return defaultValue;
218 }
219
insertCharactersIntoVector(const char * const characters,std::vector<int> * const vector)220 /* static */ void HeaderReadWriteUtils::insertCharactersIntoVector(const char *const characters,
221 std::vector<int> *const vector) {
222 for (int i = 0; characters[i]; ++i) {
223 vector->push_back(characters[i]);
224 }
225 }
226
227 } // namespace latinime
228