1# Copyright 2023 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14 15load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") 16load("@rules_cc//cc:cc_library.bzl", "cc_library") 17load("@rules_python//python:proto.bzl", "py_proto_library") 18load("@rules_python//sphinxdocs:sphinx_docs_library.bzl", "sphinx_docs_library") 19load("//pw_build:compatibility.bzl", "incompatible_with_mcu") 20load("//pw_build:copy_to_bin.bzl", "copy_to_bin") 21load("//pw_build:pw_cc_binary.bzl", "pw_cc_binary") 22load("//pw_build:pw_cc_blob_library.bzl", "pw_cc_blob_info", "pw_cc_blob_library") 23load("//pw_build:pw_linker_script.bzl", "pw_linker_script") 24load("//pw_fuzzer:fuzzer.bzl", "pw_cc_fuzz_test") 25load("//pw_unit_test:pw_cc_test.bzl", "pw_cc_test") 26 27package( 28 default_visibility = ["//visibility:public"], 29) 30 31licenses(["notice"]) 32 33cc_library( 34 name = "pw_tokenizer", 35 srcs = [ 36 "encode_args.cc", 37 "hash.cc", 38 "tokenize.cc", 39 ], 40 hdrs = [ 41 "public/pw_tokenizer/config.h", 42 "public/pw_tokenizer/encode_args.h", 43 "public/pw_tokenizer/enum.h", 44 "public/pw_tokenizer/hash.h", 45 "public/pw_tokenizer/internal/argument_types.h", 46 "public/pw_tokenizer/internal/argument_types_macro_4_byte.h", 47 "public/pw_tokenizer/internal/argument_types_macro_8_byte.h", 48 "public/pw_tokenizer/internal/enum.h", 49 "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h", 50 "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_256_hash_macro.h", 51 "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h", 52 "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h", 53 "public/pw_tokenizer/internal/tokenize_string.h", 54 "public/pw_tokenizer/nested_tokenization.h", 55 "public/pw_tokenizer/tokenize.h", 56 ], 57 strip_include_prefix = "public", 58 tags = ["noclangtidy"], 59 deps = [ 60 ":config_override", 61 "//pw_bytes:bit", 62 "//pw_containers:to_array", 63 "//pw_polyfill", 64 "//pw_preprocessor", 65 "//pw_span", 66 "//pw_varint", 67 ], 68) 69 70label_flag( 71 name = "config_override", 72 build_setting_default = "//pw_build:default_module_config", 73) 74 75pw_linker_script( 76 name = "linker_script", 77 linker_script = "pw_tokenizer_linker_sections.ld", 78) 79 80cc_library( 81 name = "test_backend", 82 visibility = ["//targets:__pkg__"], 83) 84 85cc_library( 86 name = "base64", 87 srcs = [ 88 "base64.cc", 89 ], 90 hdrs = [ 91 "public/pw_tokenizer/base64.h", 92 ], 93 strip_include_prefix = "public", 94 deps = [ 95 ":pw_tokenizer", 96 "//pw_base64", 97 "//pw_preprocessor", 98 "//pw_span", 99 "//pw_string:string", 100 ], 101) 102 103cc_library( 104 name = "decoder", 105 srcs = [ 106 "decode.cc", 107 "detokenize.cc", 108 "token_database.cc", 109 ], 110 hdrs = [ 111 "public/pw_tokenizer/detokenize.h", 112 "public/pw_tokenizer/internal/decode.h", 113 "public/pw_tokenizer/token_database.h", 114 ], 115 implementation_deps = [ 116 "//pw_elf:reader", 117 ], 118 strip_include_prefix = "public", 119 deps = [ 120 ":base64", 121 ":csv", 122 ":pw_tokenizer", 123 "//pw_bytes", 124 "//pw_bytes:bit", 125 "//pw_log", 126 "//pw_preprocessor", 127 "//pw_result", 128 "//pw_span", 129 "//pw_status", 130 "//pw_stream", 131 "//pw_varint", 132 ], 133) 134 135cc_library( 136 name = "csv", 137 srcs = ["csv.cc"], 138 hdrs = ["pw_tokenizer_private/csv.h"], 139 implementation_deps = ["//pw_log"], 140 visibility = ["//visibility:private"], 141) 142 143pw_cc_test( 144 name = "csv_test", 145 srcs = ["csv_test.cc"], 146 deps = [":csv"], 147) 148 149proto_library( 150 name = "tokenizer_proto", 151 srcs = [ 152 "pw_tokenizer_proto/options.proto", 153 ], 154 strip_import_prefix = "/pw_tokenizer", 155 deps = [ 156 "@com_google_protobuf//:descriptor_proto", 157 ], 158) 159 160py_proto_library( 161 name = "tokenizer_proto_py_pb2", 162 deps = [":tokenizer_proto"], 163) 164 165# Executable for generating test data for the C++ and Python detokenizers. This 166# target should only be built for the host. 167pw_cc_binary( 168 name = "generate_decoding_test_data", 169 srcs = [ 170 "generate_decoding_test_data.cc", 171 ], 172 target_compatible_with = select( 173 { 174 "@platforms//os:linux": [], 175 "@platforms//os:macos": [], 176 "@platforms//os:windows": [], 177 "//conditions:default": ["@platforms//:incompatible"], 178 }, 179 ), 180 deps = [ 181 ":decoder", 182 ":pw_tokenizer", 183 "//pw_preprocessor", 184 "//pw_span", 185 "//pw_varint", 186 ], 187) 188 189pw_cc_test( 190 name = "argument_types_test", 191 srcs = [ 192 "argument_types_test.cc", 193 "argument_types_test_c.c", 194 "pw_tokenizer_private/argument_types_test.h", 195 ], 196 deps = [ 197 ":pw_tokenizer", 198 "//pw_polyfill", 199 "//pw_preprocessor", 200 ], 201) 202 203pw_cc_test( 204 name = "base64_test", 205 srcs = [ 206 "base64_test.cc", 207 ], 208 deps = [ 209 ":base64", 210 "//pw_span", 211 ], 212) 213 214pw_cc_test( 215 name = "decode_test", 216 srcs = [ 217 "decode_test.cc", 218 "pw_tokenizer_private/tokenized_string_decoding_test_data.h", 219 "pw_tokenizer_private/varint_decoding_test_data.h", 220 ], 221 # TODO: https://pwbug.dev/346628514 - Fix this for rp2040 222 target_compatible_with = select({ 223 "//pw_build/constraints/chipset:rp2040": ["@platforms//:incompatible"], 224 "//conditions:default": [], 225 }), 226 deps = [ 227 ":decoder", 228 "//pw_varint", 229 ], 230) 231 232pw_cc_blob_info( 233 name = "detokenizer_example_elf_blob", 234 file_path = "//pw_tokenizer/py:example_binary_with_tokenized_strings", 235 symbol_name = "kElfSection", 236) 237 238pw_cc_blob_library( 239 name = "detokenizer_elf_test_blob", 240 blobs = [ 241 ":detokenizer_example_elf_blob", 242 ], 243 namespace = "test::ns", 244 out_header = "pw_tokenizer/example_binary_with_tokenized_strings.h", 245) 246 247pw_cc_test( 248 name = "detokenize_test", 249 srcs = [ 250 "detokenize_test.cc", 251 ], 252 deps = [ 253 ":decoder", 254 ":detokenizer_elf_test_blob", 255 "//pw_stream", 256 ], 257) 258 259pw_cc_fuzz_test( 260 name = "detokenize_fuzzer", 261 srcs = ["detokenize_fuzzer.cc"], 262 deps = [ 263 ":decoder", 264 ":pw_tokenizer", 265 "//pw_preprocessor", 266 ], 267) 268 269pw_cc_test( 270 name = "encode_args_test", 271 srcs = ["encode_args_test.cc"], 272 deps = [":pw_tokenizer"], 273) 274 275pw_cc_test( 276 name = "hash_test", 277 srcs = [ 278 "hash_test.cc", 279 "pw_tokenizer_private/generated_hash_test_cases.h", 280 ], 281 tags = ["noclangtidy"], 282 deps = [ 283 ":pw_tokenizer", 284 "//pw_preprocessor", 285 ], 286) 287 288pw_cc_test( 289 name = "simple_tokenize_test", 290 srcs = [ 291 "simple_tokenize_test.cc", 292 ], 293 deps = [":pw_tokenizer"], 294) 295 296pw_cc_test( 297 name = "token_database_test", 298 srcs = [ 299 "token_database_test.cc", 300 ], 301 deps = [":decoder"], 302) 303 304pw_cc_test( 305 name = "tokenize_test", 306 srcs = [ 307 "pw_tokenizer_private/tokenize_test.h", 308 "tokenize_test.cc", 309 "tokenize_test_c.c", 310 ], 311 # TODO: b/344050496 - get working on rp2040 and stm32f429i 312 target_compatible_with = incompatible_with_mcu(), 313 deps = [ 314 ":pw_tokenizer", 315 "//pw_preprocessor", 316 "//pw_varint", 317 ], 318) 319 320pw_cc_test( 321 name = "tokenize_c99_test", 322 srcs = ["tokenize_c99_test_entry_point.cc"], 323 deps = [":tokenize_c99_test_c"], 324) 325 326pw_cc_test( 327 name = "enum_test", 328 srcs = [ 329 "enum_test.cc", 330 ], 331 deps = [ 332 "//pw_compilation_testing:negative_compilation_testing", 333 "//pw_tokenizer", 334 ], 335) 336 337cc_library( 338 name = "tokenize_c99_test_c", 339 srcs = ["tokenize_c99_test.c"], 340 copts = [ 341 "-std=c99", 342 # pw_tokenizer uses static_assert, so this test uses a static_assert to 343 # verify that it works. Silence warnings about preadopting C11 features. 344 "-Wno-c11-extensions", 345 ], 346 visibility = ["//visibility:private"], 347 deps = [ 348 ":pw_tokenizer", 349 "//pw_containers:inline_var_len_entry_queue", 350 "//pw_varint", 351 ], 352) 353 354# Create a shared library for the tokenizer JNI wrapper. The include paths for 355# the JNI headers must be available in the system or provided with the 356# pw_java_native_interface_include_dirs variable. 357filegroup( 358 name = "detokenizer_jni", 359 srcs = [ 360 "java/dev/pigweed/tokenizer/detokenizer.cc", 361 ], 362) 363 364filegroup( 365 name = "doxygen", 366 srcs = [ 367 "public/pw_tokenizer/config.h", 368 "public/pw_tokenizer/detokenize.h", 369 "public/pw_tokenizer/encode_args.h", 370 "public/pw_tokenizer/enum.h", 371 "public/pw_tokenizer/nested_tokenization.h", 372 "public/pw_tokenizer/token_database.h", 373 "public/pw_tokenizer/tokenize.h", 374 ], 375) 376 377sphinx_docs_library( 378 name = "docs", 379 srcs = [ 380 "Kconfig", 381 "api.rst", 382 "detokenization.rst", 383 "docs.rst", 384 "enum_test.cc", 385 "get_started.rst", 386 "public/pw_tokenizer/internal/tokenize_string.h", 387 "token_databases.rst", 388 "tokenization.rst", 389 ], 390 prefix = "pw_tokenizer/", 391 target_compatible_with = incompatible_with_mcu(), 392) 393 394copy_to_bin( 395 name = "js_protos", 396 srcs = ["pw_tokenizer_proto/options.proto"], 397) 398