1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "utils/resources.h"
18
19 #include "utils/base/logging.h"
20 #include "utils/zlib/buffer_generated.h"
21 #include "utils/zlib/zlib.h"
22
23 namespace libtextclassifier3 {
24 namespace {
isWildcardMatch(const flatbuffers::String * left,const std::string & right)25 bool isWildcardMatch(const flatbuffers::String* left,
26 const std::string& right) {
27 return (left == nullptr || right.empty());
28 }
29
isExactMatch(const flatbuffers::String * left,const std::string & right)30 bool isExactMatch(const flatbuffers::String* left, const std::string& right) {
31 if (left == nullptr) {
32 return right.empty();
33 }
34 return left->str() == right;
35 }
36
37 } // namespace
38
LocaleMatch(const Locale & locale,const LanguageTag * entry_locale) const39 int Resources::LocaleMatch(const Locale& locale,
40 const LanguageTag* entry_locale) const {
41 int match = LOCALE_NO_MATCH;
42 if (isExactMatch(entry_locale->language(), locale.Language())) {
43 match |= LOCALE_LANGUAGE_MATCH;
44 } else if (isWildcardMatch(entry_locale->language(), locale.Language())) {
45 match |= LOCALE_LANGUAGE_WILDCARD_MATCH;
46 }
47
48 if (isExactMatch(entry_locale->script(), locale.Script())) {
49 match |= LOCALE_SCRIPT_MATCH;
50 } else if (isWildcardMatch(entry_locale->script(), locale.Script())) {
51 match |= LOCALE_SCRIPT_WILDCARD_MATCH;
52 }
53
54 if (isExactMatch(entry_locale->region(), locale.Region())) {
55 match |= LOCALE_REGION_MATCH;
56 } else if (isWildcardMatch(entry_locale->region(), locale.Region())) {
57 match |= LOCALE_REGION_WILDCARD_MATCH;
58 }
59
60 return match;
61 }
62
FindResource(const StringPiece resource_name) const63 const ResourceEntry* Resources::FindResource(
64 const StringPiece resource_name) const {
65 if (resources_ == nullptr || resources_->resource_entry() == nullptr) {
66 TC3_LOG(ERROR) << "No resources defined.";
67 return nullptr;
68 }
69 const ResourceEntry* entry =
70 resources_->resource_entry()->LookupByKey(resource_name.data());
71 if (entry == nullptr) {
72 TC3_LOG(ERROR) << "Resource " << resource_name.ToString() << " not found";
73 return nullptr;
74 }
75 return entry;
76 }
77
BestResourceForLocales(const ResourceEntry * resource,const std::vector<Locale> & locales) const78 int Resources::BestResourceForLocales(
79 const ResourceEntry* resource, const std::vector<Locale>& locales) const {
80 // Find best match based on locale.
81 int resource_id = -1;
82 int locale_match = LOCALE_NO_MATCH;
83 const auto* resources = resource->resource();
84 for (int user_locale = 0; user_locale < locales.size(); user_locale++) {
85 if (!locales[user_locale].IsValid()) {
86 continue;
87 }
88 for (int i = 0; i < resources->size(); i++) {
89 for (const int locale_id : *resources->Get(i)->locale()) {
90 const int candidate_match = LocaleMatch(
91 locales[user_locale], resources_->locale()->Get(locale_id));
92
93 // Only consider if at least the language matches.
94 if ((candidate_match & LOCALE_LANGUAGE_MATCH) == 0 &&
95 (candidate_match & LOCALE_LANGUAGE_WILDCARD_MATCH) == 0) {
96 continue;
97 }
98
99 if (candidate_match > locale_match) {
100 locale_match = candidate_match;
101 resource_id = i;
102 }
103 }
104 }
105
106 // If the language matches exactly, we are already finished.
107 // We found an exact language match.
108 if (locale_match & LOCALE_LANGUAGE_MATCH) {
109 return resource_id;
110 }
111 }
112 return resource_id;
113 }
114
GetResourceContent(const std::vector<Locale> & locales,const StringPiece resource_name,std::string * result) const115 bool Resources::GetResourceContent(const std::vector<Locale>& locales,
116 const StringPiece resource_name,
117 std::string* result) const {
118 const ResourceEntry* entry = FindResource(resource_name);
119 if (entry == nullptr || entry->resource() == nullptr) {
120 return false;
121 }
122
123 int resource_id = BestResourceForLocales(entry, locales);
124 if (resource_id < 0) {
125 return false;
126 }
127 const auto* resource = entry->resource()->Get(resource_id);
128 if (resource->content() != nullptr) {
129 *result = resource->content()->str();
130 return true;
131 } else if (resource->compressed_content() != nullptr) {
132 std::unique_ptr<ZlibDecompressor> decompressor = ZlibDecompressor::Instance(
133 resources_->compression_dictionary()->data(),
134 resources_->compression_dictionary()->size());
135 if (decompressor != nullptr &&
136 decompressor->MaybeDecompress(resource->compressed_content(), result)) {
137 return true;
138 }
139 }
140 return false;
141 }
142
CompressResources(ResourcePoolT * resources,const bool build_compression_dictionary,const int dictionary_sample_every)143 bool CompressResources(ResourcePoolT* resources,
144 const bool build_compression_dictionary,
145 const int dictionary_sample_every) {
146 std::vector<unsigned char> dictionary;
147 if (build_compression_dictionary) {
148 {
149 // Build up a compression dictionary.
150 std::unique_ptr<ZlibCompressor> compressor = ZlibCompressor::Instance();
151 int i = 0;
152 for (auto& entry : resources->resource_entry) {
153 for (auto& resource : entry->resource) {
154 if (resource->content.empty()) {
155 continue;
156 }
157 i++;
158
159 // Use a sample of the entries to build up a custom compression
160 // dictionary. Using all entries will generally not give a benefit
161 // for small data sizes, so we subsample here.
162 if (i % dictionary_sample_every != 0) {
163 continue;
164 }
165 CompressedBufferT compressed_content;
166 compressor->Compress(resource->content, &compressed_content);
167 }
168 }
169 compressor->GetDictionary(&dictionary);
170 resources->compression_dictionary.assign(
171 dictionary.data(), dictionary.data() + dictionary.size());
172 }
173 }
174
175 for (auto& entry : resources->resource_entry) {
176 for (auto& resource : entry->resource) {
177 if (resource->content.empty()) {
178 continue;
179 }
180 // Try compressing the data.
181 std::unique_ptr<ZlibCompressor> compressor =
182 build_compression_dictionary
183 ? ZlibCompressor::Instance(dictionary.data(), dictionary.size())
184 : ZlibCompressor::Instance();
185 if (!compressor) {
186 TC3_LOG(ERROR) << "Cannot create zlib compressor.";
187 return false;
188 }
189
190 CompressedBufferT compressed_content;
191 compressor->Compress(resource->content, &compressed_content);
192
193 // Only keep compressed version if smaller.
194 if (compressed_content.uncompressed_size >
195 compressed_content.buffer.size()) {
196 resource->content.clear();
197 resource->compressed_content.reset(new CompressedBufferT);
198 *resource->compressed_content = compressed_content;
199 }
200 }
201 }
202 return true;
203 }
204
CompressSerializedResources(const std::string & resources,const int dictionary_sample_every)205 std::string CompressSerializedResources(const std::string& resources,
206 const int dictionary_sample_every) {
207 std::unique_ptr<ResourcePoolT> unpacked_resources(
208 flatbuffers::GetRoot<ResourcePool>(resources.data())->UnPack());
209 TC3_CHECK(unpacked_resources != nullptr);
210 TC3_CHECK(
211 CompressResources(unpacked_resources.get(), dictionary_sample_every));
212 flatbuffers::FlatBufferBuilder builder;
213 builder.Finish(ResourcePool::Pack(builder, unpacked_resources.get()));
214 return std::string(reinterpret_cast<const char*>(builder.GetBufferPointer()),
215 builder.GetSize());
216 }
217
DecompressResources(ResourcePoolT * resources,const bool build_compression_dictionary)218 bool DecompressResources(ResourcePoolT* resources,
219 const bool build_compression_dictionary) {
220 std::vector<unsigned char> dictionary;
221
222 for (auto& entry : resources->resource_entry) {
223 for (auto& resource : entry->resource) {
224 if (resource->compressed_content == nullptr) {
225 continue;
226 }
227
228 std::unique_ptr<ZlibDecompressor> zlib_decompressor =
229 build_compression_dictionary
230 ? ZlibDecompressor::Instance(dictionary.data(), dictionary.size())
231 : ZlibDecompressor::Instance();
232 if (!zlib_decompressor) {
233 TC3_LOG(ERROR) << "Cannot initialize decompressor.";
234 return false;
235 }
236
237 if (!zlib_decompressor->MaybeDecompress(
238 resource->compressed_content.get(), &resource->content)) {
239 TC3_LOG(ERROR) << "Cannot decompress resource.";
240 return false;
241 }
242 resource->compressed_content.reset(nullptr);
243 }
244 }
245 return true;
246 }
247
248 } // namespace libtextclassifier3
249