1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef SRC_TRACE_PROCESSOR_UTIL_STREAMING_LINE_READER_H_ 18 #define SRC_TRACE_PROCESSOR_UTIL_STREAMING_LINE_READER_H_ 19 20 #include <cstddef> 21 #include <functional> 22 #include <vector> 23 24 #include "perfetto/ext/base/string_view.h" 25 26 namespace perfetto::trace_processor::util { 27 28 // A streaming line tokenizer for efficiently processing large text files on a 29 // line-by-line basis. It's designed to be used in conjunction with ZipReader to 30 // stream lines out of a compressed file (think of a bugreport) without having 31 // to decompress the whole file in memory upfront. 32 // Internally it deals with the necessary buffering and line-merging across 33 // different chunks. 34 // Usage: 35 // - The caller should pass a callback into the ctor. The callback is invoked 36 // whenever a batch of lines has been tokenized. This happens after calls to 37 // either BeginWrite()+EndWrite() or Tokenize(). In order to avoid too much 38 // virtual dispatch overhead, the callback argument is a vector of lines, not 39 // a single line. 40 // - The caller can call either: 41 // - Tokenize(whole input): this exist to avoid a copy in the case of 42 // non-compressed (STORE) files in zip archive. 43 // - A sequence of BeginWrite() + EndWrite() as follows: 44 // - BeginWrite(n) guarantees that the caller can write at least `n` char. 45 // `n` is typically the decompression buffer passed to zlib. 46 // - The caller writes at most `n` bytes into the pointer returned above. 47 // - The caller calls EndWrite(m) passing the number of bytes actually 48 // written (`m` <= `n`); 49 // NOTE: 50 // This implementation slightly diverges from base::StringSplitter as follows: 51 // 1. It does NOT skip empty lines. SS coalesces empty tokens, this doesn't. 52 // 2. it won't output the last line unless it terminates with a \n. SS doesn't 53 // tell the difference between "foo\nbar" and "foo\nbar\n". This is 54 // fundamental for streaming, where we cannot tell upfront if we got the end. 55 class StreamingLineReader { 56 public: 57 // Note: the lifetime of the lines passed in the vector argument is valid only 58 // for the duration of the callback. Don't retain the StringView(s) passed. 59 using LinesCallback = 60 std::function<void(const std::vector<base::StringView>&)>; 61 62 explicit StreamingLineReader(LinesCallback); 63 ~StreamingLineReader(); 64 65 // This can be used when the whole input is known upfront and we just need 66 // splitting. This exist mostly for convenience when processing uncompressed 67 // (STORE) files in zip archives. If you just need a tokenizer outside of the 68 // context of a zip file, you are better off just using base::StringSplitter. 69 size_t Tokenize(base::StringView input); 70 71 // Reserves `write_buf_size` bytes into the internal buffer. The caller is 72 // expected to write at most `write_buf_size` on the returned pointer and 73 // then call EndWrite(). 74 char* BeginWrite(size_t write_buf_size); 75 76 // Finishes the write reporting the number of bytes actually written, which 77 // must be <= `write_buf_size`. If one or more lines can be tokenized, this 78 // will cause one or more calls to the LinesCallback. 79 void EndWrite(size_t size_written); 80 81 private: 82 std::vector<char> buf_; 83 LinesCallback lines_callback_; 84 size_t size_before_write_ = 0; 85 }; 86 87 } // namespace perfetto::trace_processor::util 88 89 #endif // SRC_TRACE_PROCESSOR_UTIL_STREAMING_LINE_READER_H_ 90