| /external/tflite-support/tensorflow_lite_support/custom_ops/kernel/sentencepiece/ |
| D | model_converter.cc | 29 namespace sentencepiece { namespace
33 const ::sentencepiece::NormalizerSpec& normalizer_spec) { in DecodePrecompiledCharsmap()
52 ::sentencepiece::ModelProto model_config; in ConvertSentencepieceModelToFlatBuffer()
69 case ::sentencepiece::ModelProto::SentencePiece::NORMAL: in ConvertSentencepieceModelToFlatBuffer()
70 case ::sentencepiece::ModelProto::SentencePiece::USER_DEFINED: in ConvertSentencepieceModelToFlatBuffer()
77 case ::sentencepiece::ModelProto::SentencePiece::UNKNOWN: in ConvertSentencepieceModelToFlatBuffer()
78 case ::sentencepiece::ModelProto::SentencePiece::CONTROL: in ConvertSentencepieceModelToFlatBuffer()
130 ::sentencepiece::ModelProto model_config; in ConvertSentencepieceModelToFlatBufferForDecoder()
144 case ::sentencepiece::ModelProto::SentencePiece::NORMAL: in ConvertSentencepieceModelToFlatBufferForDecoder()
145 case ::sentencepiece::ModelProto::SentencePiece::USER_DEFINED: in ConvertSentencepieceModelToFlatBufferForDecoder()
[all …]
|
| D | sentencepiece_detokenizer_tflite.cc | 33 namespace sentencepiece { namespace
93 sentencepiece::detokenizer::Initialize, sentencepiece::detokenizer::Free, in Register_SENTENCEPIECE_DETOKENIZER()
94 sentencepiece::detokenizer::Prepare, sentencepiece::detokenizer::Eval}; in Register_SENTENCEPIECE_DETOKENIZER()
|
| D | sentencepiece_tokenizer_tflite.cc | 32 namespace sentencepiece { namespace
122 sentencepiece::tokenizer::Initialize, sentencepiece::tokenizer::Free, in Register_SENTENCEPIECE_TOKENIZER()
123 sentencepiece::tokenizer::Prepare, sentencepiece::tokenizer::Eval}; in Register_SENTENCEPIECE_TOKENIZER()
|
| D | optimized_decoder_test.cc | 30 namespace sentencepiece { namespace
67 ::sentencepiece::SentencePieceProcessor processor; in TEST()
71 ::sentencepiece::SentencePieceText reference_encoded; in TEST()
|
| D | encoder_config.fbs | 14 include "tensorflow_lite_support/custom_ops/kernel/sentencepiece/config.fbs";
16 namespace tflite.ops.custom.sentencepiece;
26 // big negative weight,less than any other sentencepiece.
|
| D | optimized_encoder_test.cc | 35 namespace sentencepiece { namespace
146 ::sentencepiece::SentencePieceProcessor processor; in TEST()
154 ::sentencepiece::SentencePieceText reference_encoded; in TEST()
|
| D | sentencepiece_detokenizer_op.cc | 70 const auto res = tflite::ops::custom::sentencepiece::DecodeString( in Compute()
75 tflite::ops::custom::sentencepiece::DecoderResultType::SUCCESS, in Compute()
|
| D | decoder_config.fbs | 16 include "tensorflow_lite_support/custom_ops/kernel/sentencepiece/config.fbs";
18 namespace tflite.ops.custom.sentencepiece;
|
| D | sentencepiece_tokenizer_op.cc | 83 const auto res = ::tflite::ops::custom::sentencepiece::EncodeString( in Compute()
88 ::tflite::ops::custom::sentencepiece::EncoderResultType::SUCCESS, in Compute()
|
| D | sentencepiece_constants.h | 21 namespace sentencepiece {
|
| D | optimized_decoder.h | 27 namespace sentencepiece {
|
| D | double_array_trie_builder.h | 28 namespace sentencepiece {
|
| D | optimized_encoder.h | 30 namespace sentencepiece {
|
| D | model_converter.h | 25 namespace sentencepiece {
|
| D | optimized_decoder.cc | 27 namespace sentencepiece { namespace
|
| D | utils.h | 25 namespace sentencepiece {
|
| D | config.fbs | 16 namespace tflite.ops.custom.sentencepiece;
|
| D | double_array_trie_builder.cc | 26 namespace sentencepiece { namespace
|
| D | double_array_trie.h | 28 namespace sentencepiece {
|
| /external/tflite-support/tensorflow_lite_support/custom_ops/python/ |
| D | BUILD | 21 "//tensorflow_lite_support/custom_ops/kernel/sentencepiece:sentencepiece_detokenizer_op.so",
22 "//tensorflow_lite_support/custom_ops/kernel/sentencepiece:sentencepiece_tokenizer_op.so",
27 "//tensorflow_lite_support/custom_ops/kernel/sentencepiece/py:pywrap_model_converter",
35 "//tensorflow_lite_support/custom_ops/kernel/sentencepiece:testdata",
42 "//tensorflow_lite_support/custom_ops/kernel/sentencepiece/py:pywrap_tflite_registerer",
|
| /external/tflite-support/ |
| D | Android.bp | 40 "tensorflow_lite_support/custom_ops/kernel/sentencepiece/sentencepiece_tokenizer_tflite.cc",
41 "tensorflow_lite_support/custom_ops/kernel/sentencepiece/optimized_encoder.cc",
88 srcs: ["tensorflow_lite_support/custom_ops/kernel/sentencepiece/config.fbs"],
89 out: ["tensorflow_lite_support/custom_ops/kernel/sentencepiece/config_generated.h"],
95 srcs: ["tensorflow_lite_support/custom_ops/kernel/sentencepiece/encoder_config.fbs"],
96 out: ["tensorflow_lite_support/custom_ops/kernel/sentencepiece/encoder_config_generated.h"],
|
| /external/oss-fuzz/projects/sentencepiece/ |
| D | Dockerfile | 19 RUN git clone --depth 1 https://github.com/google/sentencepiece.git sentencepiece
20 WORKDIR sentencepiece
|
| D | sample_encode_fuzzer.cc | 25 sentencepiece::SentencePieceProcessor fuzz_sp_processor; in LLVMFuzzerTestOneInput()
|
| /external/tflite-support/tensorflow_lite_support/cc/text/tokenizers/ |
| D | sentencepiece_tokenizer.h | 66 sentencepiece::SentencePieceProcessor sp_;
|
| /external/tflite-support/tensorflow_lite_support/opensource/ |
| D | opensource_only.files | 1 tensorflow_lite_support/custom_ops/kernel/sentencepiece/native.bzl
|