1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "ui/app_list/search/tokenized_string.h" 6 7 #include "base/i18n/break_iterator.h" 8 #include "base/i18n/case_conversion.h" 9 #include "base/logging.h" 10 #include "ui/app_list/search/term_break_iterator.h" 11 12 using base::i18n::BreakIterator; 13 14 namespace app_list { 15 TokenizedString(const base::string16 & text)16TokenizedString::TokenizedString(const base::string16& text) 17 : text_(text) { 18 Tokenize(); 19 } 20 ~TokenizedString()21TokenizedString::~TokenizedString() {} 22 Tokenize()23void TokenizedString::Tokenize() { 24 BreakIterator break_iter(text_, BreakIterator::BREAK_WORD); 25 if (!break_iter.Init()) { 26 NOTREACHED() << "BreakIterator init failed" 27 << ", text=\"" << text_ << "\""; 28 return; 29 } 30 31 while (break_iter.Advance()) { 32 if (!break_iter.IsWord()) 33 continue; 34 35 const base::string16 word(break_iter.GetString()); 36 const size_t word_start = break_iter.prev(); 37 TermBreakIterator term_iter(word); 38 while (term_iter.Advance()) { 39 tokens_.push_back(base::i18n::ToLower(term_iter.GetCurrentTerm())); 40 mappings_.push_back(gfx::Range(word_start + term_iter.prev(), 41 word_start + term_iter.pos())); 42 } 43 } 44 } 45 46 } // namespace app_list 47