1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "utils/grammar/utils/locale-shard-map.h"
18
19 #include <algorithm>
20 #include <string>
21 #include <utility>
22 #include <vector>
23
24 #include "utils/i18n/locale-list.h"
25 #include "utils/i18n/locale.h"
26 #include "utils/strings/append.h"
27
28 namespace libtextclassifier3::grammar {
29 namespace {
30
31 // For latency we put sub-rules on the first shard which must be any match
32 // i.e. '*' rules are always included while parsing the tree as it is only
33 // on shard one hence will be deduped correctly.
34 const char kAnyMatchLanguageTag[] = "*";
35
LocaleTagsToLocaleList(const std::string & locale_tags)36 std::vector<Locale> LocaleTagsToLocaleList(const std::string& locale_tags) {
37 std::vector<Locale> locale_list;
38 for (const Locale& locale : LocaleList::ParseFrom(locale_tags).GetLocales()) {
39 if (locale.IsValid()) {
40 locale_list.emplace_back(locale);
41 }
42 }
43 std::stable_sort(locale_list.begin(), locale_list.end(),
44 [](const Locale& a, const Locale& b) { return a < b; });
45 return locale_list;
46 }
47
48 } // namespace
49
CreateLocaleShardMap(const std::vector<std::string> & locale_tags,const bool include_any_match_language_tag_shard)50 LocaleShardMap LocaleShardMap::CreateLocaleShardMap(
51 const std::vector<std::string>& locale_tags,
52 const bool include_any_match_language_tag_shard) {
53 LocaleShardMap locale_shard_map;
54
55 if (include_any_match_language_tag_shard) {
56 // Make sure "*" [AnyMatch] is at shard 0.
57 locale_shard_map.AddLocalTags(kAnyMatchLanguageTag);
58 }
59
60 for (const std::string& locale_tag : locale_tags) {
61 locale_shard_map.AddLocalTags(locale_tag);
62 }
63 return locale_shard_map;
64 }
65
GetLocales(const int shard) const66 std::vector<Locale> LocaleShardMap::GetLocales(const int shard) const {
67 auto locale_it = shard_to_locale_data_.find(shard);
68 if (locale_it != shard_to_locale_data_.end()) {
69 return locale_it->second;
70 }
71 return std::vector<Locale>();
72 }
73
GetNumberOfShards() const74 int LocaleShardMap::GetNumberOfShards() const {
75 return shard_to_locale_data_.size();
76 }
77
GetShard(const std::vector<Locale> locales) const78 int LocaleShardMap::GetShard(const std::vector<Locale> locales) const {
79 for (const auto& [shard, locale_list] : shard_to_locale_data_) {
80 if (std::equal(locales.begin(), locales.end(), locale_list.begin())) {
81 return shard;
82 }
83 }
84 return 0;
85 }
86
GetShard(const std::string & locale_tags) const87 int LocaleShardMap::GetShard(const std::string& locale_tags) const {
88 std::vector<Locale> locale_list = LocaleTagsToLocaleList(locale_tags);
89 return GetShard(locale_list);
90 }
91
AddLocalTags(const std::string & locale_tags)92 void LocaleShardMap::AddLocalTags(const std::string& locale_tags) {
93 std::vector<Locale> locale_list = LocaleTagsToLocaleList(locale_tags);
94 int shard_id = shard_to_locale_data_.size();
95 shard_to_locale_data_.insert({shard_id, locale_list});
96 }
97
98 } // namespace libtextclassifier3::grammar
99