1 //
2 //
3 // Copyright 2015 gRPC authors.
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License");
6 // you may not use this file except in compliance with the License.
7 // You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
17 //
18
19 #include "src/core/ext/transport/chttp2/transport/hpack_encoder.h"
20
21 #include <grpc/event_engine/memory_allocator.h>
22 #include <grpc/slice_buffer.h>
23 #include <stdlib.h>
24 #include <string.h>
25
26 #include <memory>
27 #include <string>
28
29 #include "absl/log/log.h"
30 #include "gmock/gmock.h"
31 #include "gtest/gtest.h"
32 #include "src/core/ext/transport/chttp2/transport/legacy_frame.h"
33 #include "src/core/lib/iomgr/exec_ctx.h"
34 #include "src/core/lib/resource_quota/arena.h"
35 #include "src/core/lib/resource_quota/memory_quota.h"
36 #include "src/core/lib/resource_quota/resource_quota.h"
37 #include "src/core/util/ref_counted_ptr.h"
38 #include "test/core/test_util/parse_hexstring.h"
39 #include "test/core/test_util/slice_splitter.h"
40 #include "test/core/test_util/test_config.h"
41
42 grpc_core::HPackCompressor* g_compressor;
43
44 typedef struct {
45 bool eof;
46 bool use_true_binary_metadata;
47 } verify_params;
48
49 // verify that the output frames that are generated by encoding the stream
50 // have sensible type and flags values
verify_frames(grpc_slice_buffer & output,bool header_is_eof)51 static void verify_frames(grpc_slice_buffer& output, bool header_is_eof) {
52 // per the HTTP/2 spec:
53 // All frames begin with a fixed 9-octet header followed by a
54 // variable-length payload.
55
56 // +-----------------------------------------------+
57 // | Length (24) |
58 // +---------------+---------------+---------------+
59 // | Type (8) | Flags (8) |
60 // +-+-------------+---------------+-------------------------------+
61 // |R| Stream Identifier (31) |
62 // +=+=============================================================+
63 // | Frame Payload (0...) ...
64 // +---------------------------------------------------------------+
65 //
66 uint8_t type = 0xff, flags = 0xff;
67 size_t i, merged_length, frame_size;
68 bool first_frame = false;
69 bool in_header = false;
70 bool end_header = false;
71 bool is_closed = false;
72 for (i = 0; i < output.count;) {
73 first_frame = i == 0;
74 grpc_slice* slice = &output.slices[i++];
75
76 // Read gRPC frame header
77 uint8_t* p = GRPC_SLICE_START_PTR(*slice);
78 frame_size = 0;
79 frame_size |= static_cast<uint32_t>(p[0]) << 16;
80 frame_size |= static_cast<uint32_t>(p[1]) << 8;
81 frame_size |= static_cast<uint32_t>(p[2]);
82 type = p[3];
83 flags = p[4];
84
85 // Read remainder of the gRPC frame
86 merged_length = GRPC_SLICE_LENGTH(*slice);
87 while (merged_length < frame_size + 9) { // including 9 byte frame header
88 grpc_slice* slice = &output.slices[i++];
89 merged_length += GRPC_SLICE_LENGTH(*slice);
90 }
91
92 // Verifications
93 if (first_frame && type != GRPC_CHTTP2_FRAME_HEADER) {
94 LOG(ERROR) << "expected first frame to be of type header";
95 LOG(ERROR) << "EXPECT: " << GRPC_CHTTP2_FRAME_HEADER;
96 LOG(ERROR) << "GOT: " << type;
97 EXPECT_TRUE(false);
98 } else if (first_frame && header_is_eof &&
99 !(flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM)) {
100 LOG(ERROR) << "missing END_STREAM flag in HEADER frame";
101 EXPECT_TRUE(false);
102 }
103 if (is_closed &&
104 (type == GRPC_CHTTP2_FRAME_DATA || type == GRPC_CHTTP2_FRAME_HEADER)) {
105 LOG(ERROR)
106 << "stream is closed; new frame headers and data are not allowed";
107 EXPECT_TRUE(false);
108 }
109 if (end_header && (type == GRPC_CHTTP2_FRAME_HEADER ||
110 type == GRPC_CHTTP2_FRAME_CONTINUATION)) {
111 LOG(ERROR)
112 << "frame header is ended; new headers and continuations are not "
113 "allowed";
114 EXPECT_TRUE(false);
115 }
116 if (in_header &&
117 (type == GRPC_CHTTP2_FRAME_DATA || type == GRPC_CHTTP2_FRAME_HEADER)) {
118 LOG(ERROR)
119 << "parsing frame header; new headers and data are not allowed";
120 EXPECT_TRUE(false);
121 }
122 if (flags & ~(GRPC_CHTTP2_DATA_FLAG_END_STREAM |
123 GRPC_CHTTP2_DATA_FLAG_END_HEADERS)) {
124 LOG(ERROR) << "unexpected frame flags: " << flags;
125 EXPECT_TRUE(false);
126 }
127
128 // Update state
129 if (flags & GRPC_CHTTP2_DATA_FLAG_END_HEADERS) {
130 in_header = false;
131 end_header = true;
132 } else if (type == GRPC_CHTTP2_DATA_FLAG_END_HEADERS) {
133 in_header = true;
134 }
135 if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) {
136 is_closed = true;
137 if (type == GRPC_CHTTP2_FRAME_CONTINUATION) {
138 LOG(ERROR) << "unexpected END_STREAM flag in CONTINUATION frame";
139 EXPECT_TRUE(false);
140 }
141 }
142 }
143 }
144
CrashOnAppendError(absl::string_view,const grpc_core::Slice &)145 static void CrashOnAppendError(absl::string_view, const grpc_core::Slice&) {
146 abort();
147 }
148
149 namespace grpc_core {
150
151 class FakeCallTracer final : public CallTracerInterface {
152 public:
RecordIncomingBytes(const TransportByteSize & transport_byte_size)153 void RecordIncomingBytes(
154 const TransportByteSize& transport_byte_size) override {}
RecordOutgoingBytes(const TransportByteSize & transport_byte_size)155 void RecordOutgoingBytes(
156 const TransportByteSize& transport_byte_size) override {}
RecordSendInitialMetadata(grpc_metadata_batch * send_initial_metadata)157 void RecordSendInitialMetadata(
158 grpc_metadata_batch* send_initial_metadata) override {}
RecordSendTrailingMetadata(grpc_metadata_batch * send_trailing_metadata)159 void RecordSendTrailingMetadata(
160 grpc_metadata_batch* send_trailing_metadata) override {}
RecordSendMessage(const SliceBuffer & send_message)161 void RecordSendMessage(const SliceBuffer& send_message) override {}
RecordSendCompressedMessage(const SliceBuffer & send_compressed_message)162 void RecordSendCompressedMessage(
163 const SliceBuffer& send_compressed_message) override {}
RecordReceivedInitialMetadata(grpc_metadata_batch * recv_initial_metadata)164 void RecordReceivedInitialMetadata(
165 grpc_metadata_batch* recv_initial_metadata) override {}
RecordReceivedMessage(const SliceBuffer & recv_message)166 void RecordReceivedMessage(const SliceBuffer& recv_message) override {}
RecordReceivedDecompressedMessage(const SliceBuffer & recv_decompressed_message)167 void RecordReceivedDecompressedMessage(
168 const SliceBuffer& recv_decompressed_message) override {}
RecordCancel(grpc_error_handle cancel_error)169 void RecordCancel(grpc_error_handle cancel_error) override {}
StartNewTcpTrace()170 std::shared_ptr<TcpTracerInterface> StartNewTcpTrace() override {
171 return nullptr;
172 }
RecordAnnotation(absl::string_view annotation)173 void RecordAnnotation(absl::string_view annotation) override {}
RecordAnnotation(const Annotation & annotation)174 void RecordAnnotation(const Annotation& annotation) override {}
TraceId()175 std::string TraceId() override { return ""; }
SpanId()176 std::string SpanId() override { return ""; }
IsSampled()177 bool IsSampled() override { return false; }
178 };
179
180 } // namespace grpc_core
181
EncodeHeaderIntoBytes(bool is_eof,const std::vector<std::pair<std::string,std::string>> & header_fields)182 grpc_slice EncodeHeaderIntoBytes(
183 bool is_eof,
184 const std::vector<std::pair<std::string, std::string>>& header_fields) {
185 std::unique_ptr<grpc_core::HPackCompressor> compressor =
186 std::make_unique<grpc_core::HPackCompressor>();
187 grpc_metadata_batch b;
188
189 for (const auto& field : header_fields) {
190 b.Append(field.first,
191 grpc_core::Slice::FromStaticString(field.second.c_str()),
192 CrashOnAppendError);
193 }
194
195 grpc_core::FakeCallTracer call_tracer;
196 grpc_core::HPackCompressor::EncodeHeaderOptions hopt{
197 0xdeadbeef, // stream_id
198 is_eof, // is_eof
199 false, // use_true_binary_metadata
200 16384, // max_frame_size
201 &call_tracer};
202 grpc_slice_buffer output;
203 grpc_slice_buffer_init(&output);
204
205 compressor->EncodeHeaders(hopt, b, &output);
206 verify_frames(output, is_eof);
207
208 grpc_slice ret = grpc_slice_merge(output.slices, output.count);
209 grpc_slice_buffer_destroy(&output);
210
211 return ret;
212 }
213
214 // verify that the output generated by encoding the stream matches the
215 // hexstring passed in
verify(bool is_eof,const char * expected,const std::vector<std::pair<std::string,std::string>> & header_fields)216 static void verify(
217 bool is_eof, const char* expected,
218 const std::vector<std::pair<std::string, std::string>>& header_fields) {
219 const grpc_core::Slice merged(EncodeHeaderIntoBytes(is_eof, header_fields));
220 const grpc_core::Slice expect(grpc_core::ParseHexstring(expected));
221
222 EXPECT_EQ(merged, expect);
223 }
224
TEST(HpackEncoderTest,TestBasicHeaders)225 TEST(HpackEncoderTest, TestBasicHeaders) {
226 grpc_core::ExecCtx exec_ctx;
227 g_compressor = new grpc_core::HPackCompressor();
228
229 verify(false, "000005 0104 deadbeef 00 0161 0161", {{"a", "a"}});
230 verify(false, "00000a 0104 deadbeef 00 0161 0161 00 0162 0163",
231 {{"a", "a"}, {"b", "c"}});
232
233 delete g_compressor;
234 }
235
236 MATCHER(HasLiteralHeaderFieldNewNameFlagIncrementalIndexing, "") {
237 constexpr size_t kHttp2FrameHeaderSize = 9u;
238 /// Reference: https://httpwg.org/specs/rfc7541.html#rfc.section.6.2.1
239 /// The first byte of a literal header field with incremental indexing should
240 /// be 0x40.
241 constexpr uint8_t kLiteralHeaderFieldNewNameFlagIncrementalIndexing = 0x40;
242 return (GRPC_SLICE_START_PTR(arg)[kHttp2FrameHeaderSize] ==
243 kLiteralHeaderFieldNewNameFlagIncrementalIndexing);
244 }
245
246 MATCHER(HasLiteralHeaderFieldNewNameFlagNoIndexing, "") {
247 constexpr size_t kHttp2FrameHeaderSize = 9u;
248 /// Reference: https://httpwg.org/specs/rfc7541.html#rfc.section.6.2.2
249 /// The first byte of a literal header field without indexing should be 0x0.
250 constexpr uint8_t kLiteralHeaderFieldNewNameFlagNoIndexing = 0x00;
251 return (GRPC_SLICE_START_PTR(arg)[kHttp2FrameHeaderSize] ==
252 kLiteralHeaderFieldNewNameFlagNoIndexing);
253 }
254
TEST(HpackEncoderTest,GrpcTraceBinMetadataIndexing)255 TEST(HpackEncoderTest, GrpcTraceBinMetadataIndexing) {
256 grpc_core::ExecCtx exec_ctx;
257
258 const grpc_slice encoded_header = EncodeHeaderIntoBytes(
259 false, {{grpc_core::GrpcTraceBinMetadata::key().data(), "value"}});
260 EXPECT_THAT(encoded_header,
261 HasLiteralHeaderFieldNewNameFlagIncrementalIndexing());
262
263 grpc_slice_unref(encoded_header);
264 }
265
TEST(HpackEncoderTest,GrpcTraceBinMetadataNoIndexing)266 TEST(HpackEncoderTest, GrpcTraceBinMetadataNoIndexing) {
267 grpc_core::ExecCtx exec_ctx;
268
269 /// needs to be greater than `HPackEncoderTable::MaxEntrySize()`
270 constexpr size_t long_value_size = 70000u;
271 const grpc_slice encoded_header = EncodeHeaderIntoBytes(
272 false, {{grpc_core::GrpcTraceBinMetadata::key().data(),
273 std::string(long_value_size, 'a')}});
274 EXPECT_THAT(encoded_header, HasLiteralHeaderFieldNewNameFlagNoIndexing());
275
276 grpc_slice_unref(encoded_header);
277 }
278
TEST(HpackEncoderTest,TestGrpcTagsBinMetadataIndexing)279 TEST(HpackEncoderTest, TestGrpcTagsBinMetadataIndexing) {
280 grpc_core::ExecCtx exec_ctx;
281
282 const grpc_slice encoded_header = EncodeHeaderIntoBytes(
283 false,
284 {{grpc_core::GrpcTagsBinMetadata::key().data(), std::string("value")}});
285 EXPECT_THAT(encoded_header,
286 HasLiteralHeaderFieldNewNameFlagIncrementalIndexing());
287
288 grpc_slice_unref(encoded_header);
289 }
290
TEST(HpackEncoderTest,TestGrpcTagsBinMetadataNoIndexing)291 TEST(HpackEncoderTest, TestGrpcTagsBinMetadataNoIndexing) {
292 grpc_core::ExecCtx exec_ctx;
293
294 /// needs to be greater than `HPackEncoderTable::MaxEntrySize()`
295 constexpr size_t long_value_size = 70000u;
296 const grpc_slice encoded_header = EncodeHeaderIntoBytes(
297 false, {{grpc_core::GrpcTagsBinMetadata::key().data(),
298 std::string(long_value_size, 'a')}});
299 EXPECT_THAT(encoded_header, HasLiteralHeaderFieldNewNameFlagNoIndexing());
300
301 grpc_slice_unref(encoded_header);
302 }
303
TEST(HpackEncoderTest,UserAgentMetadataIndexing)304 TEST(HpackEncoderTest, UserAgentMetadataIndexing) {
305 grpc_core::ExecCtx exec_ctx;
306
307 const grpc_slice encoded_header = EncodeHeaderIntoBytes(
308 false, {{grpc_core::UserAgentMetadata::key().data(), "value"}});
309 EXPECT_THAT(encoded_header,
310 HasLiteralHeaderFieldNewNameFlagIncrementalIndexing());
311
312 grpc_slice_unref(encoded_header);
313 }
314
TEST(HpackEncoderTest,UserAgentMetadataNoIndexing)315 TEST(HpackEncoderTest, UserAgentMetadataNoIndexing) {
316 grpc_core::ExecCtx exec_ctx;
317
318 /// needs to be greater than `HPackEncoderTable::MaxEntrySize()`
319 constexpr size_t long_value_size = 70000u;
320 const grpc_slice encoded_header =
321 EncodeHeaderIntoBytes(false, {{grpc_core::UserAgentMetadata::key().data(),
322 std::string(long_value_size, 'a')}});
323 EXPECT_THAT(encoded_header, HasLiteralHeaderFieldNewNameFlagNoIndexing());
324
325 grpc_slice_unref(encoded_header);
326 }
327
verify_continuation_headers(const char * key,const char * value,bool is_eof)328 static void verify_continuation_headers(const char* key, const char* value,
329 bool is_eof) {
330 grpc_core::MemoryAllocator memory_allocator =
331 grpc_core::MemoryAllocator(grpc_core::ResourceQuota::Default()
332 ->memory_quota()
333 ->CreateMemoryAllocator("test"));
334 grpc_slice_buffer output;
335 grpc_metadata_batch b;
336 b.Append(key, grpc_core::Slice::FromStaticString(value), CrashOnAppendError);
337 grpc_slice_buffer_init(&output);
338
339 grpc_core::FakeCallTracer call_tracer;
340 grpc_core::HPackCompressor::EncodeHeaderOptions hopt = {
341 0xdeadbeef, // stream_id
342 is_eof, // is_eof
343 false, // use_true_binary_metadata
344 150, // max_frame_size
345 &call_tracer};
346 g_compressor->EncodeHeaders(hopt, b, &output);
347 verify_frames(output, is_eof);
348 grpc_slice_buffer_destroy(&output);
349 }
350
TEST(HpackEncoderTest,TestContinuationHeaders)351 TEST(HpackEncoderTest, TestContinuationHeaders) {
352 grpc_core::ExecCtx exec_ctx;
353 g_compressor = new grpc_core::HPackCompressor();
354
355 char value[200];
356 memset(value, 'a', 200);
357 value[199] = 0; // null terminator
358 verify_continuation_headers("key", value, true);
359
360 char value2[400];
361 memset(value2, 'b', 400);
362 value2[399] = 0; // null terminator
363 verify_continuation_headers("key2", value2, true);
364
365 delete g_compressor;
366 }
367
TEST(HpackEncoderTest,EncodeBinaryAsBase64)368 TEST(HpackEncoderTest, EncodeBinaryAsBase64) {
369 grpc_metadata_batch b;
370 // Haiku by Bard
371 b.Append("grpc-trace-bin",
372 grpc_core::Slice::FromStaticString(
373 "Base64, a tool\nTo encode binary data into "
374 "text\nSo it can be shared."),
375 CrashOnAppendError);
376 grpc_core::FakeCallTracer call_tracer;
377 grpc_slice_buffer output;
378 grpc_slice_buffer_init(&output);
379 grpc_core::HPackCompressor::EncodeHeaderOptions hopt = {
380 0xdeadbeef, // stream_id
381 true, // is_eof
382 false, // use_true_binary_metadata
383 150, // max_frame_size
384 &call_tracer};
385 grpc_core::HPackCompressor compressor;
386 compressor.EncodeHeaders(hopt, b, &output);
387 grpc_slice_buffer_destroy(&output);
388
389 EXPECT_EQ(compressor.test_only_table_size(), 136);
390 }
391
TEST(HpackEncoderTest,EncodeBinaryAsTrueBinary)392 TEST(HpackEncoderTest, EncodeBinaryAsTrueBinary) {
393 grpc_metadata_batch b;
394 // Haiku by Bard
395 b.Append("grpc-trace-bin",
396 grpc_core::Slice::FromStaticString(
397 "Base64, a tool\nTo encode binary data into "
398 "text\nSo it can be shared."),
399 CrashOnAppendError);
400 grpc_core::FakeCallTracer call_tracer;
401 grpc_slice_buffer output;
402 grpc_slice_buffer_init(&output);
403 grpc_core::HPackCompressor::EncodeHeaderOptions hopt = {
404 0xdeadbeef, // stream_id
405 true, // is_eof
406 true, // use_true_binary_metadata
407 150, // max_frame_size
408 &call_tracer};
409 grpc_core::HPackCompressor compressor;
410 compressor.EncodeHeaders(hopt, b, &output);
411 grpc_slice_buffer_destroy(&output);
412
413 EXPECT_EQ(compressor.test_only_table_size(), 114);
414 }
415
main(int argc,char ** argv)416 int main(int argc, char** argv) {
417 grpc::testing::TestEnvironment env(&argc, argv);
418 ::testing::InitGoogleTest(&argc, argv);
419 grpc::testing::TestGrpcScope grpc_scope;
420 return RUN_ALL_TESTS();
421 }
422