/* * Copyright (C) 2018 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef LIBTEXTCLASSIFIER_ANNOTATOR_NUMBER_NUMBER_H_ #define LIBTEXTCLASSIFIER_ANNOTATOR_NUMBER_NUMBER_H_ #include #include #include #include "annotator/model_generated.h" #include "annotator/types.h" #include "utils/base/logging.h" #include "utils/container/sorted-strings-table.h" #include "utils/tokenizer.h" #include "utils/utf8/unicodetext.h" namespace libtextclassifier3 { // Annotator of numbers in text. // // Integer supported values are in range [-1 000 000 000, 1 000 000 000]. // Doble supposted values are in range [-999999999.999999999, // 999999999.999999999]. class NumberAnnotator { public: explicit NumberAnnotator(const NumberAnnotatorOptions* options, const UniLib* unilib) : options_(options), unilib_(unilib), tokenizer_(Tokenizer(TokenizationType_LETTER_DIGIT, unilib, /*codepoint_ranges=*/{}, /*internal_tokenizer_codepoint_ranges=*/{}, /*split_on_script_change=*/false, /*icu_preserve_whitespace_tokens=*/true)), percent_suffixes_(FromFlatbufferStringToUnordredSet( options_->percentage_pieces_string())), max_number_of_digits_(options->max_number_of_digits()) {} // Classifies given text, and if it is a number, it passes the result in // 'classification_result' and returns true, otherwise returns false. bool ClassifyText(const UnicodeText& context, CodepointSpan selection_indices, AnnotationUsecase annotation_usecase, ClassificationResult* classification_result) const; // Finds all number instances in the input text. Returns true in any case. bool FindAll(const UnicodeText& context_unicode, AnnotationUsecase annotation_usecase, std::vector* result) const; private: // Converts a Flatbuffer string containing zero-separated percent suffixes // to an unordered set. static std::unordered_set FromFlatbufferStringToUnordredSet( const flatbuffers::String* flatbuffer_percent_strings); // Checks if the annotated numbers from the context represent percentages. // If yes, replaces the collection type and the annotation boundary in the // result. void FindPercentages(const UnicodeText& context, std::vector* result) const; // Checks if the tokens from in the interval [start_index-2, start_index] are // valid characters that can preced a number context. bool TokensAreValidStart(const std::vector& tokens, int start_index) const; // Checks if the tokens in the interval (..., prefix_end_index] are a valid // number prefix. bool TokensAreValidNumberPrefix(const std::vector& tokens, int prefix_end_index) const; // Checks if the tokens from in the interval [ending_index, ending_index+2] // are valid characters that can follow a number context. bool TokensAreValidEnding(const std::vector& tokens, int ending_index) const; // Checks if the tokens in the interval [suffix_start_index, ...) are a valid // number suffix. bool TokensAreValidNumberSuffix(const std::vector& tokens, int suffix_start_index) const; // Checks if the tokens in the interval [suffix_start_index, ...) are a valid // percent suffix. If false, returns -1, else returns the end codepoint. int FindPercentSuffixEndCodepoint(const std::vector& tokens, int suffix_token_start_index) const; // Checks if the given text represents a number (either int or double). bool TryParseNumber(const UnicodeText& token_text, bool is_negative, int64* parsed_int_value, double* parsed_double_value) const; // Checks if a word contains only CJT characters. bool IsCJTterm(UnicodeText::const_iterator token_begin_it, int token_length) const; AnnotatedSpan CreateAnnotatedSpan(int start, int end, int int_value, double double_value, const std::string collection, float score, float priority_score) const; const NumberAnnotatorOptions* options_; const UniLib* unilib_; const Tokenizer tokenizer_; const std::unordered_set percent_suffixes_; const int max_number_of_digits_; }; } // namespace libtextclassifier3 #endif // LIBTEXTCLASSIFIER_ANNOTATOR_NUMBER_NUMBER_H_