1 /*
2 * nghttp2 - HTTP/2 C Library
3 *
4 * Copyright (c) 2012 Tatsuhiro Tsujikawa
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25 #include "shrpx_http2_session.h"
26
27 #include <netinet/tcp.h>
28 #ifdef HAVE_UNISTD_H
29 # include <unistd.h>
30 #endif // HAVE_UNISTD_H
31
32 #include <vector>
33
34 #include "ssl_compat.h"
35
36 #ifdef NGHTTP2_OPENSSL_IS_WOLFSSL
37 # include <wolfssl/options.h>
38 # include <wolfssl/openssl/err.h>
39 #else // !NGHTTP2_OPENSSL_IS_WOLFSSL
40 # include <openssl/err.h>
41 #endif // !NGHTTP2_OPENSSL_IS_WOLFSSL
42
43 #include "shrpx_upstream.h"
44 #include "shrpx_downstream.h"
45 #include "shrpx_config.h"
46 #include "shrpx_error.h"
47 #include "shrpx_http2_downstream_connection.h"
48 #include "shrpx_client_handler.h"
49 #include "shrpx_tls.h"
50 #include "shrpx_http.h"
51 #include "shrpx_worker.h"
52 #include "shrpx_connect_blocker.h"
53 #include "shrpx_log.h"
54 #include "http2.h"
55 #include "util.h"
56 #include "base64.h"
57 #include "tls.h"
58
59 using namespace nghttp2;
60
61 namespace shrpx {
62
63 namespace {
64 constexpr ev_tstamp CONNCHK_TIMEOUT = 5.;
65 constexpr ev_tstamp CONNCHK_PING_TIMEOUT = 1.;
66 } // namespace
67
68 namespace {
69 constexpr size_t MAX_BUFFER_SIZE = 32_k;
70 } // namespace
71
72 namespace {
connchk_timeout_cb(struct ev_loop * loop,ev_timer * w,int revents)73 void connchk_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
74 auto http2session = static_cast<Http2Session *>(w->data);
75
76 ev_timer_stop(loop, w);
77
78 switch (http2session->get_connection_check_state()) {
79 case ConnectionCheck::STARTED:
80 // ping timeout; disconnect
81 if (LOG_ENABLED(INFO)) {
82 SSLOG(INFO, http2session) << "ping timeout";
83 }
84
85 delete http2session;
86
87 return;
88 default:
89 if (LOG_ENABLED(INFO)) {
90 SSLOG(INFO, http2session) << "connection check required";
91 }
92 http2session->set_connection_check_state(ConnectionCheck::REQUIRED);
93 }
94 }
95 } // namespace
96
97 namespace {
settings_timeout_cb(struct ev_loop * loop,ev_timer * w,int revents)98 void settings_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
99 auto http2session = static_cast<Http2Session *>(w->data);
100
101 if (LOG_ENABLED(INFO)) {
102 SSLOG(INFO, http2session) << "SETTINGS timeout";
103 }
104
105 downstream_failure(http2session->get_addr(), http2session->get_raddr());
106
107 if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) {
108 delete http2session;
109
110 return;
111 }
112 http2session->signal_write();
113 }
114 } // namespace
115
116 namespace {
timeoutcb(struct ev_loop * loop,ev_timer * w,int revents)117 void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
118 auto conn = static_cast<Connection *>(w->data);
119 auto http2session = static_cast<Http2Session *>(conn->data);
120
121 if (w == &conn->rt && !conn->expired_rt()) {
122 return;
123 }
124
125 if (LOG_ENABLED(INFO)) {
126 SSLOG(INFO, http2session) << "Timeout";
127 }
128
129 http2session->on_timeout();
130
131 delete http2session;
132 }
133 } // namespace
134
135 namespace {
readcb(struct ev_loop * loop,ev_io * w,int revents)136 void readcb(struct ev_loop *loop, ev_io *w, int revents) {
137 int rv;
138 auto conn = static_cast<Connection *>(w->data);
139 auto http2session = static_cast<Http2Session *>(conn->data);
140 rv = http2session->do_read();
141 if (rv != 0) {
142 delete http2session;
143
144 return;
145 }
146 http2session->connection_alive();
147 }
148 } // namespace
149
150 namespace {
writecb(struct ev_loop * loop,ev_io * w,int revents)151 void writecb(struct ev_loop *loop, ev_io *w, int revents) {
152 int rv;
153 auto conn = static_cast<Connection *>(w->data);
154 auto http2session = static_cast<Http2Session *>(conn->data);
155 rv = http2session->do_write();
156 if (rv != 0) {
157 delete http2session;
158
159 return;
160 }
161 http2session->reset_connection_check_timer_if_not_checking();
162 }
163 } // namespace
164
165 namespace {
initiate_connection_cb(struct ev_loop * loop,ev_timer * w,int revents)166 void initiate_connection_cb(struct ev_loop *loop, ev_timer *w, int revents) {
167 auto http2session = static_cast<Http2Session *>(w->data);
168 ev_timer_stop(loop, w);
169 if (http2session->initiate_connection() != 0) {
170 if (LOG_ENABLED(INFO)) {
171 SSLOG(INFO, http2session) << "Could not initiate backend connection";
172 }
173
174 delete http2session;
175
176 return;
177 }
178 }
179 } // namespace
180
181 namespace {
prepare_cb(struct ev_loop * loop,ev_prepare * w,int revents)182 void prepare_cb(struct ev_loop *loop, ev_prepare *w, int revents) {
183 auto http2session = static_cast<Http2Session *>(w->data);
184 http2session->check_retire();
185 }
186 } // namespace
187
Http2Session(struct ev_loop * loop,SSL_CTX * ssl_ctx,Worker * worker,const std::shared_ptr<DownstreamAddrGroup> & group,DownstreamAddr * addr)188 Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
189 Worker *worker,
190 const std::shared_ptr<DownstreamAddrGroup> &group,
191 DownstreamAddr *addr)
192 : dlnext(nullptr),
193 dlprev(nullptr),
194 conn_(loop, -1, nullptr, worker->get_mcpool(),
195 group->shared_addr->timeout.write, group->shared_addr->timeout.read,
196 {}, {}, writecb, readcb, timeoutcb, this,
197 get_config()->tls.dyn_rec.warmup_threshold,
198 get_config()->tls.dyn_rec.idle_timeout, Proto::HTTP2),
199 wb_(worker->get_mcpool()),
200 worker_(worker),
201 ssl_ctx_(ssl_ctx),
202 group_(group),
203 addr_(addr),
204 session_(nullptr),
205 raddr_(nullptr),
206 state_(Http2SessionState::DISCONNECTED),
207 connection_check_state_(ConnectionCheck::NONE),
208 freelist_zone_(FreelistZone::NONE),
209 settings_recved_(false),
210 allow_connect_proto_(false) {
211 read_ = write_ = &Http2Session::noop;
212
213 on_read_ = &Http2Session::read_noop;
214 on_write_ = &Http2Session::write_noop;
215
216 // We will reuse this many times, so use repeat timeout value. The
217 // timeout value is set later.
218 ev_timer_init(&connchk_timer_, connchk_timeout_cb, 0., 0.);
219
220 connchk_timer_.data = this;
221
222 // SETTINGS ACK timeout is 10 seconds for now. We will reuse this
223 // many times, so use repeat timeout value.
224 ev_timer_init(&settings_timer_, settings_timeout_cb, 0., 0.);
225
226 settings_timer_.data = this;
227
228 ev_timer_init(&initiate_connection_timer_, initiate_connection_cb, 0., 0.);
229 initiate_connection_timer_.data = this;
230
231 ev_prepare_init(&prep_, prepare_cb);
232 prep_.data = this;
233 ev_prepare_start(loop, &prep_);
234 }
235
~Http2Session()236 Http2Session::~Http2Session() {
237 exclude_from_scheduling();
238 disconnect(should_hard_fail());
239 }
240
disconnect(bool hard)241 int Http2Session::disconnect(bool hard) {
242 if (LOG_ENABLED(INFO)) {
243 SSLOG(INFO, this) << "Disconnecting";
244 }
245 nghttp2_session_del(session_);
246 session_ = nullptr;
247
248 wb_.reset();
249
250 if (dns_query_) {
251 auto dns_tracker = worker_->get_dns_tracker();
252 dns_tracker->cancel(dns_query_.get());
253 }
254
255 conn_.rlimit.stopw();
256 conn_.wlimit.stopw();
257
258 ev_prepare_stop(conn_.loop, &prep_);
259
260 ev_timer_stop(conn_.loop, &initiate_connection_timer_);
261 ev_timer_stop(conn_.loop, &settings_timer_);
262 ev_timer_stop(conn_.loop, &connchk_timer_);
263
264 read_ = write_ = &Http2Session::noop;
265
266 on_read_ = &Http2Session::read_noop;
267 on_write_ = &Http2Session::write_noop;
268
269 conn_.disconnect();
270
271 if (proxy_htp_) {
272 proxy_htp_.reset();
273 }
274
275 connection_check_state_ = ConnectionCheck::NONE;
276 state_ = Http2SessionState::DISCONNECTED;
277
278 // When deleting Http2DownstreamConnection, it calls this object's
279 // remove_downstream_connection(). The multiple
280 // Http2DownstreamConnection objects belong to the same
281 // ClientHandler object if upstream is h2. So be careful when you
282 // delete ClientHandler here.
283 //
284 // We allow creating new pending Http2DownstreamConnection with this
285 // object. Upstream::on_downstream_reset() may add
286 // Http2DownstreamConnection to another Http2Session.
287
288 for (auto dc = dconns_.head; dc;) {
289 auto next = dc->dlnext;
290 auto downstream = dc->get_downstream();
291 auto upstream = downstream->get_upstream();
292
293 // Failure is allowed only for HTTP/1 upstream where upstream is
294 // not shared by multiple Downstreams.
295 if (upstream->on_downstream_reset(downstream, hard) != 0) {
296 delete upstream->get_client_handler();
297 }
298
299 // dc was deleted
300 dc = next;
301 }
302
303 auto streams = std::move(streams_);
304 for (auto s = streams.head; s;) {
305 auto next = s->dlnext;
306 delete s;
307 s = next;
308 }
309
310 return 0;
311 }
312
resolve_name()313 int Http2Session::resolve_name() {
314 auto dns_query = std::make_unique<DNSQuery>(
315 addr_->host, [this](DNSResolverStatus status, const Address *result) {
316 int rv;
317
318 if (status == DNSResolverStatus::OK) {
319 *resolved_addr_ = *result;
320 util::set_port(*this->resolved_addr_, this->addr_->port);
321 }
322
323 rv = this->initiate_connection();
324 if (rv != 0) {
325 delete this;
326 }
327 });
328 resolved_addr_ = std::make_unique<Address>();
329 auto dns_tracker = worker_->get_dns_tracker();
330 switch (dns_tracker->resolve(resolved_addr_.get(), dns_query.get())) {
331 case DNSResolverStatus::ERROR:
332 return -1;
333 case DNSResolverStatus::RUNNING:
334 dns_query_ = std::move(dns_query);
335 state_ = Http2SessionState::RESOLVING_NAME;
336 return 0;
337 case DNSResolverStatus::OK:
338 util::set_port(*resolved_addr_, addr_->port);
339 return 0;
340 default:
341 assert(0);
342 abort();
343 }
344 }
345
346 namespace {
347 int htp_hdrs_completecb(llhttp_t *htp);
348 } // namespace
349
350 namespace {
351 constexpr llhttp_settings_t htp_hooks = {
352 nullptr, // llhttp_cb on_message_begin;
353 nullptr, // llhttp_data_cb on_url;
354 nullptr, // llhttp_data_cb on_status;
355 nullptr, // llhttp_data_cb on_method;
356 nullptr, // llhttp_data_cb on_version;
357 nullptr, // llhttp_data_cb on_header_field;
358 nullptr, // llhttp_data_cb on_header_value;
359 nullptr, // llhttp_data_cb on_chunk_extension_name;
360 nullptr, // llhttp_data_cb on_chunk_extension_value;
361 htp_hdrs_completecb, // llhttp_cb on_headers_complete;
362 nullptr, // llhttp_data_cb on_body;
363 nullptr, // llhttp_cb on_message_complete;
364 nullptr, // llhttp_cb on_url_complete;
365 nullptr, // llhttp_cb on_status_complete;
366 nullptr, // llhttp_cb on_method_complete;
367 nullptr, // llhttp_cb on_version_complete;
368 nullptr, // llhttp_cb on_header_field_complete;
369 nullptr, // llhttp_cb on_header_value_complete;
370 nullptr, // llhttp_cb on_chunk_extension_name_complete;
371 nullptr, // llhttp_cb on_chunk_extension_value_complete;
372 nullptr, // llhttp_cb on_chunk_header;
373 nullptr, // llhttp_cb on_chunk_complete;
374 nullptr, // llhttp_cb on_reset;
375 };
376 } // namespace
377
initiate_connection()378 int Http2Session::initiate_connection() {
379 int rv = 0;
380
381 auto worker_blocker = worker_->get_connect_blocker();
382
383 if (state_ == Http2SessionState::DISCONNECTED ||
384 state_ == Http2SessionState::RESOLVING_NAME) {
385 if (worker_blocker->blocked()) {
386 if (LOG_ENABLED(INFO)) {
387 SSLOG(INFO, this)
388 << "Worker wide backend connection was blocked temporarily";
389 }
390 return -1;
391 }
392 }
393
394 auto &downstreamconf = *get_config()->conn.downstream;
395
396 const auto &proxy = get_config()->downstream_http_proxy;
397 if (!proxy.host.empty() && state_ == Http2SessionState::DISCONNECTED) {
398 if (LOG_ENABLED(INFO)) {
399 SSLOG(INFO, this) << "Connecting to the proxy " << proxy.host << ":"
400 << proxy.port;
401 }
402
403 conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family);
404
405 if (conn_.fd == -1) {
406 auto error = errno;
407 SSLOG(WARN, this) << "Backend proxy socket() failed; addr="
408 << util::to_numeric_addr(&proxy.addr)
409 << ", errno=" << error;
410
411 worker_blocker->on_failure();
412 return -1;
413 }
414
415 rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len);
416 if (rv != 0 && errno != EINPROGRESS) {
417 auto error = errno;
418 SSLOG(WARN, this) << "Backend proxy connect() failed; addr="
419 << util::to_numeric_addr(&proxy.addr)
420 << ", errno=" << error;
421
422 worker_blocker->on_failure();
423
424 return -1;
425 }
426
427 raddr_ = &proxy.addr;
428
429 worker_blocker->on_success();
430
431 ev_io_set(&conn_.rev, conn_.fd, EV_READ);
432 ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
433
434 conn_.wlimit.startw();
435
436 conn_.wt.repeat = downstreamconf.timeout.connect;
437 ev_timer_again(conn_.loop, &conn_.wt);
438
439 write_ = &Http2Session::connected;
440
441 on_read_ = &Http2Session::downstream_read_proxy;
442 on_write_ = &Http2Session::downstream_connect_proxy;
443
444 proxy_htp_ = std::make_unique<llhttp_t>();
445 llhttp_init(proxy_htp_.get(), HTTP_RESPONSE, &htp_hooks);
446 proxy_htp_->data = this;
447
448 state_ = Http2SessionState::PROXY_CONNECTING;
449
450 return 0;
451 }
452
453 if (state_ == Http2SessionState::DISCONNECTED ||
454 state_ == Http2SessionState::PROXY_CONNECTED ||
455 state_ == Http2SessionState::RESOLVING_NAME) {
456 if (LOG_ENABLED(INFO)) {
457 if (state_ != Http2SessionState::RESOLVING_NAME) {
458 SSLOG(INFO, this) << "Connecting to downstream server";
459 }
460 }
461 if (addr_->tls) {
462 assert(ssl_ctx_);
463
464 if (state_ != Http2SessionState::RESOLVING_NAME) {
465 auto ssl = tls::create_ssl(ssl_ctx_);
466 if (!ssl) {
467 return -1;
468 }
469
470 tls::setup_downstream_http2_alpn(ssl);
471
472 conn_.set_ssl(ssl);
473 conn_.tls.client_session_cache = &addr_->tls_session_cache;
474
475 auto sni_name =
476 addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni};
477
478 if (!util::numeric_host(sni_name.data())) {
479 // TLS extensions: SNI. There is no documentation about the return
480 // code for this function (actually this is macro wrapping SSL_ctrl
481 // at the time of this writing).
482 SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.data());
483 }
484
485 auto tls_session = tls::reuse_tls_session(addr_->tls_session_cache);
486 if (tls_session) {
487 SSL_set_session(conn_.tls.ssl, tls_session);
488 SSL_SESSION_free(tls_session);
489 }
490 }
491
492 if (state_ == Http2SessionState::DISCONNECTED) {
493 if (addr_->dns) {
494 rv = resolve_name();
495 if (rv != 0) {
496 downstream_failure(addr_, nullptr);
497 return -1;
498 }
499 if (state_ == Http2SessionState::RESOLVING_NAME) {
500 return 0;
501 }
502 raddr_ = resolved_addr_.get();
503 } else {
504 raddr_ = &addr_->addr;
505 }
506 }
507
508 if (state_ == Http2SessionState::RESOLVING_NAME) {
509 if (dns_query_->status == DNSResolverStatus::ERROR) {
510 downstream_failure(addr_, nullptr);
511 return -1;
512 }
513 assert(dns_query_->status == DNSResolverStatus::OK);
514 state_ = Http2SessionState::DISCONNECTED;
515 dns_query_.reset();
516 raddr_ = resolved_addr_.get();
517 }
518
519 // If state_ == Http2SessionState::PROXY_CONNECTED, we have
520 // connected to the proxy using conn_.fd and tunnel has been
521 // established.
522 if (state_ == Http2SessionState::DISCONNECTED) {
523 assert(conn_.fd == -1);
524
525 conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
526 if (conn_.fd == -1) {
527 auto error = errno;
528 SSLOG(WARN, this)
529 << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
530 << ", errno=" << error;
531
532 worker_blocker->on_failure();
533 return -1;
534 }
535
536 worker_blocker->on_success();
537
538 rv = connect(conn_.fd,
539 // TODO maybe not thread-safe?
540 const_cast<sockaddr *>(&raddr_->su.sa), raddr_->len);
541 if (rv != 0 && errno != EINPROGRESS) {
542 auto error = errno;
543 SSLOG(WARN, this)
544 << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
545 << ", errno=" << error;
546
547 downstream_failure(addr_, raddr_);
548 return -1;
549 }
550
551 ev_io_set(&conn_.rev, conn_.fd, EV_READ);
552 ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
553 }
554
555 conn_.prepare_client_handshake();
556 } else {
557 if (state_ == Http2SessionState::DISCONNECTED) {
558 // Without TLS and proxy.
559 if (addr_->dns) {
560 rv = resolve_name();
561 if (rv != 0) {
562 downstream_failure(addr_, nullptr);
563 return -1;
564 }
565 if (state_ == Http2SessionState::RESOLVING_NAME) {
566 return 0;
567 }
568 raddr_ = resolved_addr_.get();
569 } else {
570 raddr_ = &addr_->addr;
571 }
572 }
573
574 if (state_ == Http2SessionState::RESOLVING_NAME) {
575 if (dns_query_->status == DNSResolverStatus::ERROR) {
576 downstream_failure(addr_, nullptr);
577 return -1;
578 }
579 assert(dns_query_->status == DNSResolverStatus::OK);
580 state_ = Http2SessionState::DISCONNECTED;
581 dns_query_.reset();
582 raddr_ = resolved_addr_.get();
583 }
584
585 if (state_ == Http2SessionState::DISCONNECTED) {
586 // Without TLS and proxy.
587 assert(conn_.fd == -1);
588
589 conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
590
591 if (conn_.fd == -1) {
592 auto error = errno;
593 SSLOG(WARN, this)
594 << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
595 << ", errno=" << error;
596
597 worker_blocker->on_failure();
598 return -1;
599 }
600
601 worker_blocker->on_success();
602
603 rv = connect(conn_.fd, const_cast<sockaddr *>(&raddr_->su.sa),
604 raddr_->len);
605 if (rv != 0 && errno != EINPROGRESS) {
606 auto error = errno;
607 SSLOG(WARN, this)
608 << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
609 << ", errno=" << error;
610
611 downstream_failure(addr_, raddr_);
612 return -1;
613 }
614
615 ev_io_set(&conn_.rev, conn_.fd, EV_READ);
616 ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
617 }
618 }
619
620 // We have been already connected when no TLS and proxy is used.
621 if (state_ == Http2SessionState::PROXY_CONNECTED) {
622 on_read_ = &Http2Session::read_noop;
623 on_write_ = &Http2Session::write_noop;
624
625 return connected();
626 }
627
628 write_ = &Http2Session::connected;
629
630 state_ = Http2SessionState::CONNECTING;
631 conn_.wlimit.startw();
632
633 conn_.wt.repeat = downstreamconf.timeout.connect;
634 ev_timer_again(conn_.loop, &conn_.wt);
635
636 return 0;
637 }
638
639 // Unreachable
640 assert(0);
641
642 return 0;
643 }
644
645 namespace {
htp_hdrs_completecb(llhttp_t * htp)646 int htp_hdrs_completecb(llhttp_t *htp) {
647 auto http2session = static_cast<Http2Session *>(htp->data);
648
649 // We only read HTTP header part. If tunneling succeeds, response
650 // body is a different protocol (HTTP/2 in this case), we don't read
651 // them here.
652
653 // We just check status code here
654 if (htp->status_code / 100 == 2) {
655 if (LOG_ENABLED(INFO)) {
656 SSLOG(INFO, http2session) << "Tunneling success";
657 }
658 http2session->set_state(Http2SessionState::PROXY_CONNECTED);
659
660 return HPE_PAUSED;
661 }
662
663 SSLOG(WARN, http2session) << "Tunneling failed: " << htp->status_code;
664 http2session->set_state(Http2SessionState::PROXY_FAILED);
665
666 return HPE_PAUSED;
667 }
668 } // namespace
669
downstream_read_proxy(const uint8_t * data,size_t datalen)670 int Http2Session::downstream_read_proxy(const uint8_t *data, size_t datalen) {
671 auto htperr = llhttp_execute(proxy_htp_.get(),
672 reinterpret_cast<const char *>(data), datalen);
673 if (htperr == HPE_PAUSED) {
674 switch (state_) {
675 case Http2SessionState::PROXY_CONNECTED:
676 // Initiate SSL/TLS handshake through established tunnel.
677 if (initiate_connection() != 0) {
678 return -1;
679 }
680 return 0;
681 case Http2SessionState::PROXY_FAILED:
682 return -1;
683 default:
684 break;
685 }
686 // should not be here
687 assert(0);
688 }
689
690 if (htperr != HPE_OK) {
691 return -1;
692 }
693
694 return 0;
695 }
696
downstream_connect_proxy()697 int Http2Session::downstream_connect_proxy() {
698 if (LOG_ENABLED(INFO)) {
699 SSLOG(INFO, this) << "Connected to the proxy";
700 }
701
702 std::string req = "CONNECT ";
703 req.append(addr_->hostport.data(), addr_->hostport.size());
704 if (addr_->port == 80 || addr_->port == 443) {
705 req += ':';
706 req += util::utos(addr_->port);
707 }
708 req += " HTTP/1.1\r\nHost: ";
709 req += addr_->host;
710 req += "\r\n";
711 const auto &proxy = get_config()->downstream_http_proxy;
712 if (!proxy.userinfo.empty()) {
713 req += "Proxy-Authorization: Basic ";
714 req += base64::encode(std::begin(proxy.userinfo), std::end(proxy.userinfo));
715 req += "\r\n";
716 }
717 req += "\r\n";
718 if (LOG_ENABLED(INFO)) {
719 SSLOG(INFO, this) << "HTTP proxy request headers\n" << req;
720 }
721 wb_.append(req);
722
723 on_write_ = &Http2Session::write_noop;
724
725 signal_write();
726 return 0;
727 }
728
add_downstream_connection(Http2DownstreamConnection * dconn)729 void Http2Session::add_downstream_connection(Http2DownstreamConnection *dconn) {
730 dconns_.append(dconn);
731 ++addr_->num_dconn;
732 }
733
remove_downstream_connection(Http2DownstreamConnection * dconn)734 void Http2Session::remove_downstream_connection(
735 Http2DownstreamConnection *dconn) {
736 --addr_->num_dconn;
737 dconns_.remove(dconn);
738 dconn->detach_stream_data();
739
740 if (LOG_ENABLED(INFO)) {
741 SSLOG(INFO, this) << "Remove downstream";
742 }
743
744 if (freelist_zone_ == FreelistZone::NONE && !max_concurrency_reached()) {
745 if (LOG_ENABLED(INFO)) {
746 SSLOG(INFO, this) << "Append to http2_extra_freelist, addr=" << addr_
747 << ", freelist.size="
748 << addr_->http2_extra_freelist.size();
749 }
750
751 add_to_extra_freelist();
752 }
753 }
754
remove_stream_data(StreamData * sd)755 void Http2Session::remove_stream_data(StreamData *sd) {
756 streams_.remove(sd);
757 if (sd->dconn) {
758 sd->dconn->detach_stream_data();
759 }
760 delete sd;
761 }
762
submit_request(Http2DownstreamConnection * dconn,const nghttp2_nv * nva,size_t nvlen,const nghttp2_data_provider2 * data_prd)763 int Http2Session::submit_request(Http2DownstreamConnection *dconn,
764 const nghttp2_nv *nva, size_t nvlen,
765 const nghttp2_data_provider2 *data_prd) {
766 assert(state_ == Http2SessionState::CONNECTED);
767 auto sd = std::make_unique<StreamData>();
768 sd->dlnext = sd->dlprev = nullptr;
769 // TODO Specify nullptr to pri_spec for now
770 auto stream_id =
771 nghttp2_submit_request2(session_, nullptr, nva, nvlen, data_prd, sd.get());
772 if (stream_id < 0) {
773 SSLOG(FATAL, this) << "nghttp2_submit_request2() failed: "
774 << nghttp2_strerror(stream_id);
775 return -1;
776 }
777
778 dconn->attach_stream_data(sd.get());
779 dconn->get_downstream()->set_downstream_stream_id(stream_id);
780 streams_.append(sd.release());
781
782 return 0;
783 }
784
submit_rst_stream(int32_t stream_id,uint32_t error_code)785 int Http2Session::submit_rst_stream(int32_t stream_id, uint32_t error_code) {
786 assert(state_ == Http2SessionState::CONNECTED);
787 if (LOG_ENABLED(INFO)) {
788 SSLOG(INFO, this) << "RST_STREAM stream_id=" << stream_id
789 << " with error_code=" << error_code;
790 }
791 int rv = nghttp2_submit_rst_stream(session_, NGHTTP2_FLAG_NONE, stream_id,
792 error_code);
793 if (rv != 0) {
794 SSLOG(FATAL, this) << "nghttp2_submit_rst_stream() failed: "
795 << nghttp2_strerror(rv);
796 return -1;
797 }
798 return 0;
799 }
800
get_session() const801 nghttp2_session *Http2Session::get_session() const { return session_; }
802
resume_data(Http2DownstreamConnection * dconn)803 int Http2Session::resume_data(Http2DownstreamConnection *dconn) {
804 assert(state_ == Http2SessionState::CONNECTED);
805 auto downstream = dconn->get_downstream();
806 int rv = nghttp2_session_resume_data(session_,
807 downstream->get_downstream_stream_id());
808 switch (rv) {
809 case 0:
810 case NGHTTP2_ERR_INVALID_ARGUMENT:
811 return 0;
812 default:
813 SSLOG(FATAL, this) << "nghttp2_resume_session() failed: "
814 << nghttp2_strerror(rv);
815 return -1;
816 }
817 }
818
819 namespace {
call_downstream_readcb(Http2Session * http2session,Downstream * downstream)820 void call_downstream_readcb(Http2Session *http2session,
821 Downstream *downstream) {
822 auto upstream = downstream->get_upstream();
823 if (!upstream) {
824 return;
825 }
826 if (upstream->downstream_read(downstream->get_downstream_connection()) != 0) {
827 delete upstream->get_client_handler();
828 }
829 }
830 } // namespace
831
832 namespace {
on_stream_close_callback(nghttp2_session * session,int32_t stream_id,uint32_t error_code,void * user_data)833 int on_stream_close_callback(nghttp2_session *session, int32_t stream_id,
834 uint32_t error_code, void *user_data) {
835 auto http2session = static_cast<Http2Session *>(user_data);
836 if (LOG_ENABLED(INFO)) {
837 SSLOG(INFO, http2session)
838 << "Stream stream_id=" << stream_id << " is being closed with error code "
839 << error_code;
840 }
841 auto sd = static_cast<StreamData *>(
842 nghttp2_session_get_stream_user_data(session, stream_id));
843 if (sd == 0) {
844 // We might get this close callback when pushed streams are
845 // closed.
846 return 0;
847 }
848 auto dconn = sd->dconn;
849 if (dconn) {
850 auto downstream = dconn->get_downstream();
851 auto upstream = downstream->get_upstream();
852
853 if (downstream->get_downstream_stream_id() % 2 == 0 &&
854 downstream->get_request_state() == DownstreamState::INITIAL) {
855 // Downstream is canceled in backend before it is submitted in
856 // frontend session.
857
858 // This will avoid to send RST_STREAM to backend
859 downstream->set_response_state(DownstreamState::MSG_RESET);
860 upstream->cancel_premature_downstream(downstream);
861 } else {
862 if (downstream->get_upgraded() && downstream->get_response_state() ==
863 DownstreamState::HEADER_COMPLETE) {
864 // For tunneled connection, we have to submit RST_STREAM to
865 // upstream *after* whole response body is sent. We just set
866 // MSG_COMPLETE here. Upstream will take care of that.
867 downstream->get_upstream()->on_downstream_body_complete(downstream);
868 downstream->set_response_state(DownstreamState::MSG_COMPLETE);
869 } else if (error_code == NGHTTP2_NO_ERROR) {
870 switch (downstream->get_response_state()) {
871 case DownstreamState::MSG_COMPLETE:
872 case DownstreamState::MSG_BAD_HEADER:
873 break;
874 default:
875 downstream->set_response_state(DownstreamState::MSG_RESET);
876 }
877 } else if (downstream->get_response_state() !=
878 DownstreamState::MSG_BAD_HEADER) {
879 downstream->set_response_state(DownstreamState::MSG_RESET);
880 }
881 if (downstream->get_response_state() == DownstreamState::MSG_RESET &&
882 downstream->get_response_rst_stream_error_code() ==
883 NGHTTP2_NO_ERROR) {
884 downstream->set_response_rst_stream_error_code(error_code);
885 }
886 call_downstream_readcb(http2session, downstream);
887 }
888 // dconn may be deleted
889 }
890 // The life time of StreamData ends here
891 http2session->remove_stream_data(sd);
892 return 0;
893 }
894 } // namespace
895
start_settings_timer()896 void Http2Session::start_settings_timer() {
897 auto &downstreamconf = get_config()->http2.downstream;
898
899 ev_timer_set(&settings_timer_, downstreamconf.timeout.settings, 0.);
900 ev_timer_start(conn_.loop, &settings_timer_);
901 }
902
stop_settings_timer()903 void Http2Session::stop_settings_timer() {
904 ev_timer_stop(conn_.loop, &settings_timer_);
905 }
906
907 namespace {
on_header_callback2(nghttp2_session * session,const nghttp2_frame * frame,nghttp2_rcbuf * name,nghttp2_rcbuf * value,uint8_t flags,void * user_data)908 int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame,
909 nghttp2_rcbuf *name, nghttp2_rcbuf *value,
910 uint8_t flags, void *user_data) {
911 auto http2session = static_cast<Http2Session *>(user_data);
912 auto sd = static_cast<StreamData *>(
913 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
914 if (!sd || !sd->dconn) {
915 return 0;
916 }
917 auto downstream = sd->dconn->get_downstream();
918
919 auto namebuf = nghttp2_rcbuf_get_buf(name);
920 auto valuebuf = nghttp2_rcbuf_get_buf(value);
921
922 auto &resp = downstream->response();
923 auto &httpconf = get_config()->http;
924
925 switch (frame->hd.type) {
926 case NGHTTP2_HEADERS: {
927 auto trailer = frame->headers.cat == NGHTTP2_HCAT_HEADERS &&
928 !downstream->get_expect_final_response();
929
930 if (resp.fs.buffer_size() + namebuf.len + valuebuf.len >
931 httpconf.response_header_field_buffer ||
932 resp.fs.num_fields() >= httpconf.max_response_header_fields) {
933 if (LOG_ENABLED(INFO)) {
934 DLOG(INFO, downstream)
935 << "Too large or many header field size="
936 << resp.fs.buffer_size() + namebuf.len + valuebuf.len
937 << ", num=" << resp.fs.num_fields() + 1;
938 }
939
940 if (trailer) {
941 // We don't care trailer part exceeds header size limit; just
942 // discard it.
943 return 0;
944 }
945
946 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
947 }
948
949 auto nameref = StringRef{namebuf.base, namebuf.len};
950 auto valueref = StringRef{valuebuf.base, valuebuf.len};
951 auto token = http2::lookup_token(nameref);
952 auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX;
953
954 downstream->add_rcbuf(name);
955 downstream->add_rcbuf(value);
956
957 if (trailer) {
958 // just store header fields for trailer part
959 resp.fs.add_trailer_token(nameref, valueref, no_index, token);
960 return 0;
961 }
962
963 resp.fs.add_header_token(nameref, valueref, no_index, token);
964 return 0;
965 }
966 case NGHTTP2_PUSH_PROMISE: {
967 auto promised_stream_id = frame->push_promise.promised_stream_id;
968 auto promised_sd = static_cast<StreamData *>(
969 nghttp2_session_get_stream_user_data(session, promised_stream_id));
970 if (!promised_sd || !promised_sd->dconn) {
971 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
972 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
973 }
974
975 auto promised_downstream = promised_sd->dconn->get_downstream();
976
977 auto namebuf = nghttp2_rcbuf_get_buf(name);
978 auto valuebuf = nghttp2_rcbuf_get_buf(value);
979
980 assert(promised_downstream);
981
982 auto &promised_req = promised_downstream->request();
983
984 // We use request header limit for PUSH_PROMISE
985 if (promised_req.fs.buffer_size() + namebuf.len + valuebuf.len >
986 httpconf.request_header_field_buffer ||
987 promised_req.fs.num_fields() >= httpconf.max_request_header_fields) {
988 if (LOG_ENABLED(INFO)) {
989 DLOG(INFO, downstream)
990 << "Too large or many header field size="
991 << promised_req.fs.buffer_size() + namebuf.len + valuebuf.len
992 << ", num=" << promised_req.fs.num_fields() + 1;
993 }
994
995 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
996 }
997
998 promised_downstream->add_rcbuf(name);
999 promised_downstream->add_rcbuf(value);
1000
1001 auto nameref = StringRef{namebuf.base, namebuf.len};
1002 auto valueref = StringRef{valuebuf.base, valuebuf.len};
1003 auto token = http2::lookup_token(nameref);
1004 promised_req.fs.add_header_token(nameref, valueref,
1005 flags & NGHTTP2_NV_FLAG_NO_INDEX, token);
1006
1007 return 0;
1008 }
1009 }
1010
1011 return 0;
1012 }
1013 } // namespace
1014
1015 namespace {
on_invalid_header_callback2(nghttp2_session * session,const nghttp2_frame * frame,nghttp2_rcbuf * name,nghttp2_rcbuf * value,uint8_t flags,void * user_data)1016 int on_invalid_header_callback2(nghttp2_session *session,
1017 const nghttp2_frame *frame, nghttp2_rcbuf *name,
1018 nghttp2_rcbuf *value, uint8_t flags,
1019 void *user_data) {
1020 auto http2session = static_cast<Http2Session *>(user_data);
1021 auto sd = static_cast<StreamData *>(
1022 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1023 if (!sd || !sd->dconn) {
1024 return 0;
1025 }
1026
1027 int32_t stream_id;
1028
1029 if (frame->hd.type == NGHTTP2_PUSH_PROMISE) {
1030 stream_id = frame->push_promise.promised_stream_id;
1031 } else {
1032 stream_id = frame->hd.stream_id;
1033 }
1034
1035 if (LOG_ENABLED(INFO)) {
1036 auto namebuf = nghttp2_rcbuf_get_buf(name);
1037 auto valuebuf = nghttp2_rcbuf_get_buf(value);
1038
1039 SSLOG(INFO, http2session)
1040 << "Invalid header field for stream_id=" << stream_id
1041 << " in frame type=" << static_cast<uint32_t>(frame->hd.type)
1042 << ": name=[" << StringRef{namebuf.base, namebuf.len} << "], value=["
1043 << StringRef{valuebuf.base, valuebuf.len} << "]";
1044 }
1045
1046 http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
1047
1048 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1049 }
1050 } // namespace
1051
1052 namespace {
on_begin_headers_callback(nghttp2_session * session,const nghttp2_frame * frame,void * user_data)1053 int on_begin_headers_callback(nghttp2_session *session,
1054 const nghttp2_frame *frame, void *user_data) {
1055 auto http2session = static_cast<Http2Session *>(user_data);
1056
1057 switch (frame->hd.type) {
1058 case NGHTTP2_HEADERS: {
1059 if (frame->headers.cat != NGHTTP2_HCAT_RESPONSE &&
1060 frame->headers.cat != NGHTTP2_HCAT_PUSH_RESPONSE) {
1061 return 0;
1062 }
1063 auto sd = static_cast<StreamData *>(
1064 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1065 if (!sd || !sd->dconn) {
1066 http2session->submit_rst_stream(frame->hd.stream_id,
1067 NGHTTP2_INTERNAL_ERROR);
1068 return 0;
1069 }
1070 return 0;
1071 }
1072 case NGHTTP2_PUSH_PROMISE: {
1073 auto promised_stream_id = frame->push_promise.promised_stream_id;
1074 auto sd = static_cast<StreamData *>(
1075 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1076 if (!sd || !sd->dconn) {
1077 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1078 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1079 }
1080
1081 auto downstream = sd->dconn->get_downstream();
1082
1083 assert(downstream);
1084 assert(downstream->get_downstream_stream_id() == frame->hd.stream_id);
1085
1086 if (http2session->handle_downstream_push_promise(downstream,
1087 promised_stream_id) != 0) {
1088 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1089 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1090 }
1091
1092 return 0;
1093 }
1094 }
1095
1096 return 0;
1097 }
1098 } // namespace
1099
1100 namespace {
on_response_headers(Http2Session * http2session,Downstream * downstream,nghttp2_session * session,const nghttp2_frame * frame)1101 int on_response_headers(Http2Session *http2session, Downstream *downstream,
1102 nghttp2_session *session, const nghttp2_frame *frame) {
1103 int rv;
1104
1105 auto upstream = downstream->get_upstream();
1106 auto handler = upstream->get_client_handler();
1107 const auto &req = downstream->request();
1108 auto &resp = downstream->response();
1109
1110 auto &nva = resp.fs.headers();
1111
1112 auto config = get_config();
1113 auto &loggingconf = config->logging;
1114
1115 downstream->set_expect_final_response(false);
1116
1117 auto status = resp.fs.header(http2::HD__STATUS);
1118 // libnghttp2 guarantees this exists and can be parsed
1119 assert(status);
1120 auto status_code = http2::parse_http_status_code(status->value);
1121
1122 resp.http_status = status_code;
1123 resp.http_major = 2;
1124 resp.http_minor = 0;
1125
1126 downstream->set_downstream_addr_group(
1127 http2session->get_downstream_addr_group());
1128 downstream->set_addr(http2session->get_addr());
1129
1130 if (LOG_ENABLED(INFO)) {
1131 std::stringstream ss;
1132 for (auto &nv : nva) {
1133 ss << TTY_HTTP_HD << nv.name << TTY_RST << ": " << nv.value << "\n";
1134 }
1135 SSLOG(INFO, http2session)
1136 << "HTTP response headers. stream_id=" << frame->hd.stream_id << "\n"
1137 << ss.str();
1138 }
1139
1140 if (downstream->get_non_final_response()) {
1141 if (LOG_ENABLED(INFO)) {
1142 SSLOG(INFO, http2session) << "This is non-final response.";
1143 }
1144
1145 downstream->set_expect_final_response(true);
1146 rv = upstream->on_downstream_header_complete(downstream);
1147
1148 // Now Dowstream's response headers are erased.
1149
1150 if (rv != 0) {
1151 http2session->submit_rst_stream(frame->hd.stream_id,
1152 NGHTTP2_PROTOCOL_ERROR);
1153 downstream->set_response_state(DownstreamState::MSG_RESET);
1154 }
1155
1156 return 0;
1157 }
1158
1159 downstream->set_response_state(DownstreamState::HEADER_COMPLETE);
1160 downstream->check_upgrade_fulfilled_http2();
1161
1162 if (downstream->get_upgraded()) {
1163 resp.connection_close = true;
1164 // On upgrade success, both ends can send data
1165 if (upstream->resume_read(SHRPX_NO_BUFFER, downstream, 0) != 0) {
1166 // If resume_read fails, just drop connection. Not ideal.
1167 delete handler;
1168 return -1;
1169 }
1170 downstream->set_request_state(DownstreamState::HEADER_COMPLETE);
1171 if (LOG_ENABLED(INFO)) {
1172 SSLOG(INFO, http2session)
1173 << "HTTP upgrade success. stream_id=" << frame->hd.stream_id;
1174 }
1175 } else {
1176 auto content_length = resp.fs.header(http2::HD_CONTENT_LENGTH);
1177 if (content_length) {
1178 // libnghttp2 guarantees this can be parsed
1179 resp.fs.content_length =
1180 util::parse_uint(content_length->value).value_or(-1);
1181 }
1182
1183 if (resp.fs.content_length == -1 && downstream->expect_response_body()) {
1184 // Here we have response body but Content-Length is not known in
1185 // advance.
1186 if (req.http_major <= 0 || (req.http_major == 1 && req.http_minor == 0)) {
1187 // We simply close connection for pre-HTTP/1.1 in this case.
1188 resp.connection_close = true;
1189 } else {
1190 // Otherwise, use chunked encoding to keep upstream connection
1191 // open. In HTTP2, we are supposed not to receive
1192 // transfer-encoding.
1193 resp.fs.add_header_token("transfer-encoding"_sr, "chunked"_sr, false,
1194 http2::HD_TRANSFER_ENCODING);
1195 downstream->set_chunked_response(true);
1196 }
1197 }
1198 }
1199
1200 if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1201 resp.headers_only = true;
1202 }
1203
1204 if (loggingconf.access.write_early && downstream->accesslog_ready()) {
1205 handler->write_accesslog(downstream);
1206 downstream->set_accesslog_written(true);
1207 }
1208
1209 rv = upstream->on_downstream_header_complete(downstream);
1210 if (rv != 0) {
1211 // Handling early return (in other words, response was hijacked by
1212 // mruby scripting).
1213 if (downstream->get_response_state() == DownstreamState::MSG_COMPLETE) {
1214 http2session->submit_rst_stream(frame->hd.stream_id, NGHTTP2_CANCEL);
1215 } else {
1216 http2session->submit_rst_stream(frame->hd.stream_id,
1217 NGHTTP2_INTERNAL_ERROR);
1218 downstream->set_response_state(DownstreamState::MSG_RESET);
1219 }
1220 }
1221
1222 return 0;
1223 }
1224 } // namespace
1225
1226 namespace {
on_frame_recv_callback(nghttp2_session * session,const nghttp2_frame * frame,void * user_data)1227 int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame,
1228 void *user_data) {
1229 int rv;
1230 auto http2session = static_cast<Http2Session *>(user_data);
1231
1232 switch (frame->hd.type) {
1233 case NGHTTP2_DATA: {
1234 auto sd = static_cast<StreamData *>(
1235 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1236 if (!sd || !sd->dconn) {
1237 return 0;
1238 }
1239 auto downstream = sd->dconn->get_downstream();
1240 auto upstream = downstream->get_upstream();
1241 rv = upstream->on_downstream_body(downstream, nullptr, 0, true);
1242 if (rv != 0) {
1243 http2session->submit_rst_stream(frame->hd.stream_id,
1244 NGHTTP2_INTERNAL_ERROR);
1245 downstream->set_response_state(DownstreamState::MSG_RESET);
1246
1247 } else if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1248 downstream->disable_downstream_rtimer();
1249
1250 if (downstream->get_response_state() ==
1251 DownstreamState::HEADER_COMPLETE) {
1252 downstream->set_response_state(DownstreamState::MSG_COMPLETE);
1253
1254 rv = upstream->on_downstream_body_complete(downstream);
1255
1256 if (rv != 0) {
1257 downstream->set_response_state(DownstreamState::MSG_RESET);
1258 }
1259 }
1260 }
1261
1262 call_downstream_readcb(http2session, downstream);
1263 return 0;
1264 }
1265 case NGHTTP2_HEADERS: {
1266 auto sd = static_cast<StreamData *>(
1267 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1268 if (!sd || !sd->dconn) {
1269 return 0;
1270 }
1271 auto downstream = sd->dconn->get_downstream();
1272
1273 if (frame->headers.cat == NGHTTP2_HCAT_RESPONSE ||
1274 frame->headers.cat == NGHTTP2_HCAT_PUSH_RESPONSE) {
1275 rv = on_response_headers(http2session, downstream, session, frame);
1276
1277 if (rv != 0) {
1278 return 0;
1279 }
1280 } else if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) {
1281 if (downstream->get_expect_final_response()) {
1282 rv = on_response_headers(http2session, downstream, session, frame);
1283
1284 if (rv != 0) {
1285 return 0;
1286 }
1287 }
1288 }
1289
1290 if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1291 downstream->disable_downstream_rtimer();
1292
1293 if (downstream->get_response_state() ==
1294 DownstreamState::HEADER_COMPLETE) {
1295 downstream->set_response_state(DownstreamState::MSG_COMPLETE);
1296
1297 auto upstream = downstream->get_upstream();
1298
1299 rv = upstream->on_downstream_body_complete(downstream);
1300
1301 if (rv != 0) {
1302 downstream->set_response_state(DownstreamState::MSG_RESET);
1303 }
1304 }
1305 } else {
1306 downstream->reset_downstream_rtimer();
1307 }
1308
1309 // This may delete downstream
1310 call_downstream_readcb(http2session, downstream);
1311
1312 return 0;
1313 }
1314 case NGHTTP2_RST_STREAM: {
1315 auto sd = static_cast<StreamData *>(
1316 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1317 if (sd && sd->dconn) {
1318 auto downstream = sd->dconn->get_downstream();
1319 downstream->set_response_rst_stream_error_code(
1320 frame->rst_stream.error_code);
1321 call_downstream_readcb(http2session, downstream);
1322 }
1323 return 0;
1324 }
1325 case NGHTTP2_SETTINGS: {
1326 if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
1327 http2session->on_settings_received(frame);
1328 return 0;
1329 }
1330
1331 http2session->stop_settings_timer();
1332
1333 auto addr = http2session->get_addr();
1334 auto &connect_blocker = addr->connect_blocker;
1335
1336 connect_blocker->on_success();
1337
1338 return 0;
1339 }
1340 case NGHTTP2_PING:
1341 if (frame->hd.flags & NGHTTP2_FLAG_ACK) {
1342 if (LOG_ENABLED(INFO)) {
1343 LOG(INFO) << "PING ACK received";
1344 }
1345 http2session->connection_alive();
1346 }
1347 return 0;
1348 case NGHTTP2_PUSH_PROMISE: {
1349 auto promised_stream_id = frame->push_promise.promised_stream_id;
1350
1351 if (LOG_ENABLED(INFO)) {
1352 SSLOG(INFO, http2session)
1353 << "Received downstream PUSH_PROMISE stream_id=" << frame->hd.stream_id
1354 << ", promised_stream_id=" << promised_stream_id;
1355 }
1356
1357 auto sd = static_cast<StreamData *>(
1358 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1359 if (!sd || !sd->dconn) {
1360 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1361 return 0;
1362 }
1363
1364 auto downstream = sd->dconn->get_downstream();
1365
1366 assert(downstream);
1367 assert(downstream->get_downstream_stream_id() == frame->hd.stream_id);
1368
1369 auto promised_sd = static_cast<StreamData *>(
1370 nghttp2_session_get_stream_user_data(session, promised_stream_id));
1371 if (!promised_sd || !promised_sd->dconn) {
1372 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1373 return 0;
1374 }
1375
1376 auto promised_downstream = promised_sd->dconn->get_downstream();
1377
1378 assert(promised_downstream);
1379
1380 if (http2session->handle_downstream_push_promise_complete(
1381 downstream, promised_downstream) != 0) {
1382 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1383 return 0;
1384 }
1385
1386 return 0;
1387 }
1388 case NGHTTP2_GOAWAY:
1389 if (LOG_ENABLED(INFO)) {
1390 auto debug_data = util::ascii_dump(frame->goaway.opaque_data,
1391 frame->goaway.opaque_data_len);
1392
1393 SSLOG(INFO, http2session)
1394 << "GOAWAY received: last-stream-id=" << frame->goaway.last_stream_id
1395 << ", error_code=" << frame->goaway.error_code
1396 << ", debug_data=" << debug_data;
1397 }
1398 return 0;
1399 default:
1400 return 0;
1401 }
1402 }
1403 } // namespace
1404
1405 namespace {
on_data_chunk_recv_callback(nghttp2_session * session,uint8_t flags,int32_t stream_id,const uint8_t * data,size_t len,void * user_data)1406 int on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags,
1407 int32_t stream_id, const uint8_t *data,
1408 size_t len, void *user_data) {
1409 int rv;
1410 auto http2session = static_cast<Http2Session *>(user_data);
1411 auto sd = static_cast<StreamData *>(
1412 nghttp2_session_get_stream_user_data(session, stream_id));
1413 if (!sd || !sd->dconn) {
1414 http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1415
1416 if (http2session->consume(stream_id, len) != 0) {
1417 return NGHTTP2_ERR_CALLBACK_FAILURE;
1418 }
1419
1420 return 0;
1421 }
1422 auto downstream = sd->dconn->get_downstream();
1423 if (!downstream->expect_response_body()) {
1424 http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1425
1426 if (http2session->consume(stream_id, len) != 0) {
1427 return NGHTTP2_ERR_CALLBACK_FAILURE;
1428 }
1429
1430 return 0;
1431 }
1432
1433 // We don't want DATA after non-final response, which is illegal in
1434 // HTTP.
1435 if (downstream->get_non_final_response()) {
1436 http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
1437
1438 if (http2session->consume(stream_id, len) != 0) {
1439 return NGHTTP2_ERR_CALLBACK_FAILURE;
1440 }
1441
1442 return 0;
1443 }
1444
1445 downstream->reset_downstream_rtimer();
1446
1447 auto &resp = downstream->response();
1448
1449 resp.recv_body_length += len;
1450 resp.unconsumed_body_length += len;
1451
1452 auto upstream = downstream->get_upstream();
1453 rv = upstream->on_downstream_body(downstream, data, len, false);
1454 if (rv != 0) {
1455 http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1456
1457 if (http2session->consume(stream_id, len) != 0) {
1458 return NGHTTP2_ERR_CALLBACK_FAILURE;
1459 }
1460
1461 downstream->set_response_state(DownstreamState::MSG_RESET);
1462 }
1463
1464 call_downstream_readcb(http2session, downstream);
1465 return 0;
1466 }
1467 } // namespace
1468
1469 namespace {
on_frame_send_callback(nghttp2_session * session,const nghttp2_frame * frame,void * user_data)1470 int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
1471 void *user_data) {
1472 auto http2session = static_cast<Http2Session *>(user_data);
1473
1474 if (frame->hd.type == NGHTTP2_DATA || frame->hd.type == NGHTTP2_HEADERS) {
1475 auto sd = static_cast<StreamData *>(
1476 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1477
1478 if (!sd || !sd->dconn) {
1479 return 0;
1480 }
1481
1482 auto downstream = sd->dconn->get_downstream();
1483
1484 if (frame->hd.type == NGHTTP2_HEADERS &&
1485 frame->headers.cat == NGHTTP2_HCAT_REQUEST) {
1486 downstream->set_request_header_sent(true);
1487 auto src = downstream->get_blocked_request_buf();
1488 if (src->rleft()) {
1489 auto dest = downstream->get_request_buf();
1490 src->remove(*dest);
1491 if (http2session->resume_data(sd->dconn) != 0) {
1492 return NGHTTP2_ERR_CALLBACK_FAILURE;
1493 }
1494 downstream->ensure_downstream_wtimer();
1495 }
1496 }
1497
1498 if ((frame->hd.flags & NGHTTP2_FLAG_END_STREAM) == 0) {
1499 return 0;
1500 }
1501
1502 downstream->reset_downstream_rtimer();
1503
1504 return 0;
1505 }
1506
1507 if (frame->hd.type == NGHTTP2_SETTINGS &&
1508 (frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
1509 http2session->start_settings_timer();
1510 }
1511 return 0;
1512 }
1513 } // namespace
1514
1515 namespace {
on_frame_not_send_callback(nghttp2_session * session,const nghttp2_frame * frame,int lib_error_code,void * user_data)1516 int on_frame_not_send_callback(nghttp2_session *session,
1517 const nghttp2_frame *frame, int lib_error_code,
1518 void *user_data) {
1519 auto http2session = static_cast<Http2Session *>(user_data);
1520 if (LOG_ENABLED(INFO)) {
1521 SSLOG(INFO, http2session) << "Failed to send control frame type="
1522 << static_cast<uint32_t>(frame->hd.type)
1523 << ", lib_error_code=" << lib_error_code << ": "
1524 << nghttp2_strerror(lib_error_code);
1525 }
1526 if (frame->hd.type != NGHTTP2_HEADERS ||
1527 lib_error_code == NGHTTP2_ERR_STREAM_CLOSED ||
1528 lib_error_code == NGHTTP2_ERR_STREAM_CLOSING) {
1529 return 0;
1530 }
1531
1532 auto sd = static_cast<StreamData *>(
1533 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1534 if (!sd) {
1535 return 0;
1536 }
1537 if (!sd->dconn) {
1538 return 0;
1539 }
1540 auto downstream = sd->dconn->get_downstream();
1541
1542 if (lib_error_code == NGHTTP2_ERR_START_STREAM_NOT_ALLOWED) {
1543 // Migrate to another downstream connection.
1544 auto upstream = downstream->get_upstream();
1545
1546 if (upstream->on_downstream_reset(downstream, false)) {
1547 // This should be done for h1 upstream only. Deleting
1548 // ClientHandler for h2 upstream may lead to crash.
1549 delete upstream->get_client_handler();
1550 }
1551
1552 return 0;
1553 }
1554
1555 // To avoid stream hanging around, flag DownstreamState::MSG_RESET.
1556 downstream->set_response_state(DownstreamState::MSG_RESET);
1557 call_downstream_readcb(http2session, downstream);
1558
1559 return 0;
1560 }
1561 } // namespace
1562
1563 namespace {
1564 constexpr auto PADDING = std::array<uint8_t, 256>{};
1565 } // namespace
1566
1567 namespace {
send_data_callback(nghttp2_session * session,nghttp2_frame * frame,const uint8_t * framehd,size_t length,nghttp2_data_source * source,void * user_data)1568 int send_data_callback(nghttp2_session *session, nghttp2_frame *frame,
1569 const uint8_t *framehd, size_t length,
1570 nghttp2_data_source *source, void *user_data) {
1571 auto http2session = static_cast<Http2Session *>(user_data);
1572 auto sd = static_cast<StreamData *>(
1573 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1574
1575 if (sd == nullptr) {
1576 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1577 }
1578
1579 auto dconn = sd->dconn;
1580 auto downstream = dconn->get_downstream();
1581 auto input = downstream->get_request_buf();
1582 auto wb = http2session->get_request_buf();
1583
1584 size_t padlen = 0;
1585
1586 wb->append(framehd, 9);
1587 if (frame->data.padlen > 0) {
1588 padlen = frame->data.padlen - 1;
1589 wb->append(static_cast<uint8_t>(padlen));
1590 }
1591
1592 input->remove(*wb, length);
1593
1594 wb->append(PADDING.data(), padlen);
1595
1596 if (input->rleft() == 0) {
1597 downstream->disable_downstream_wtimer();
1598 } else {
1599 downstream->reset_downstream_wtimer();
1600 }
1601
1602 if (length > 0) {
1603 // This is important because it will handle flow control
1604 // stuff.
1605 if (downstream->get_upstream()->resume_read(SHRPX_NO_BUFFER, downstream,
1606 length) != 0) {
1607 // In this case, downstream may be deleted.
1608 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1609 }
1610
1611 // Here sd->dconn could be nullptr, because
1612 // Upstream::resume_read() may delete downstream which will delete
1613 // dconn. Is this still really true?
1614 }
1615
1616 return 0;
1617 }
1618 } // namespace
1619
create_http2_downstream_callbacks()1620 nghttp2_session_callbacks *create_http2_downstream_callbacks() {
1621 int rv;
1622 nghttp2_session_callbacks *callbacks;
1623
1624 rv = nghttp2_session_callbacks_new(&callbacks);
1625
1626 if (rv != 0) {
1627 return nullptr;
1628 }
1629
1630 nghttp2_session_callbacks_set_on_stream_close_callback(
1631 callbacks, on_stream_close_callback);
1632
1633 nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks,
1634 on_frame_recv_callback);
1635
1636 nghttp2_session_callbacks_set_on_data_chunk_recv_callback(
1637 callbacks, on_data_chunk_recv_callback);
1638
1639 nghttp2_session_callbacks_set_on_frame_send_callback(callbacks,
1640 on_frame_send_callback);
1641
1642 nghttp2_session_callbacks_set_on_frame_not_send_callback(
1643 callbacks, on_frame_not_send_callback);
1644
1645 nghttp2_session_callbacks_set_on_header_callback2(callbacks,
1646 on_header_callback2);
1647
1648 nghttp2_session_callbacks_set_on_invalid_header_callback2(
1649 callbacks, on_invalid_header_callback2);
1650
1651 nghttp2_session_callbacks_set_on_begin_headers_callback(
1652 callbacks, on_begin_headers_callback);
1653
1654 nghttp2_session_callbacks_set_send_data_callback(callbacks,
1655 send_data_callback);
1656
1657 if (get_config()->padding) {
1658 nghttp2_session_callbacks_set_select_padding_callback2(
1659 callbacks, http::select_padding_callback);
1660 }
1661
1662 return callbacks;
1663 }
1664
connection_made()1665 int Http2Session::connection_made() {
1666 int rv;
1667
1668 state_ = Http2SessionState::CONNECTED;
1669
1670 on_write_ = &Http2Session::downstream_write;
1671 on_read_ = &Http2Session::downstream_read;
1672
1673 if (addr_->tls) {
1674 const unsigned char *next_proto = nullptr;
1675 unsigned int next_proto_len = 0;
1676
1677 SSL_get0_alpn_selected(conn_.tls.ssl, &next_proto, &next_proto_len);
1678
1679 if (!next_proto) {
1680 downstream_failure(addr_, raddr_);
1681 return -1;
1682 }
1683
1684 auto proto = StringRef{next_proto, next_proto_len};
1685 if (LOG_ENABLED(INFO)) {
1686 SSLOG(INFO, this) << "Negotiated next protocol: " << proto;
1687 }
1688 if (!util::check_h2_is_selected(proto)) {
1689 downstream_failure(addr_, raddr_);
1690 return -1;
1691 }
1692 }
1693
1694 auto config = get_config();
1695 auto &http2conf = config->http2;
1696
1697 rv = nghttp2_session_client_new2(&session_, http2conf.downstream.callbacks,
1698 this, http2conf.downstream.option);
1699
1700 if (rv != 0) {
1701 return -1;
1702 }
1703
1704 std::array<nghttp2_settings_entry, 5> entry;
1705 size_t nentry = 3;
1706 entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
1707 entry[0].value = http2conf.downstream.max_concurrent_streams;
1708
1709 entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
1710 entry[1].value = http2conf.downstream.window_size;
1711
1712 entry[2].settings_id = NGHTTP2_SETTINGS_NO_RFC7540_PRIORITIES;
1713 entry[2].value = 1;
1714
1715 if (http2conf.no_server_push || config->http2_proxy) {
1716 entry[nentry].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
1717 entry[nentry].value = 0;
1718 ++nentry;
1719 }
1720
1721 if (http2conf.downstream.decoder_dynamic_table_size !=
1722 NGHTTP2_DEFAULT_HEADER_TABLE_SIZE) {
1723 entry[nentry].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
1724 entry[nentry].value = http2conf.downstream.decoder_dynamic_table_size;
1725 ++nentry;
1726 }
1727
1728 rv =
1729 nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, entry.data(), nentry);
1730 if (rv != 0) {
1731 return -1;
1732 }
1733
1734 rv = nghttp2_session_set_local_window_size(
1735 session_, NGHTTP2_FLAG_NONE, 0,
1736 http2conf.downstream.connection_window_size);
1737 if (rv != 0) {
1738 return -1;
1739 }
1740
1741 reset_connection_check_timer(CONNCHK_TIMEOUT);
1742
1743 submit_pending_requests();
1744
1745 signal_write();
1746 return 0;
1747 }
1748
do_read()1749 int Http2Session::do_read() { return read_(*this); }
do_write()1750 int Http2Session::do_write() { return write_(*this); }
1751
on_read(const uint8_t * data,size_t datalen)1752 int Http2Session::on_read(const uint8_t *data, size_t datalen) {
1753 return on_read_(*this, data, datalen);
1754 }
1755
on_write()1756 int Http2Session::on_write() { return on_write_(*this); }
1757
downstream_read(const uint8_t * data,size_t datalen)1758 int Http2Session::downstream_read(const uint8_t *data, size_t datalen) {
1759 auto rv = nghttp2_session_mem_recv2(session_, data, datalen);
1760 if (rv < 0) {
1761 SSLOG(ERROR, this) << "nghttp2_session_mem_recv2() returned error: "
1762 << nghttp2_strerror(rv);
1763 return -1;
1764 }
1765
1766 if (nghttp2_session_want_read(session_) == 0 &&
1767 nghttp2_session_want_write(session_) == 0 && wb_.rleft() == 0) {
1768 if (LOG_ENABLED(INFO)) {
1769 SSLOG(INFO, this) << "No more read/write for this HTTP2 session";
1770 }
1771 return -1;
1772 }
1773
1774 signal_write();
1775 return 0;
1776 }
1777
downstream_write()1778 int Http2Session::downstream_write() {
1779 for (;;) {
1780 const uint8_t *data;
1781 auto datalen = nghttp2_session_mem_send2(session_, &data);
1782 if (datalen < 0) {
1783 SSLOG(ERROR, this) << "nghttp2_session_mem_send2() returned error: "
1784 << nghttp2_strerror(datalen);
1785 return -1;
1786 }
1787 if (datalen == 0) {
1788 break;
1789 }
1790 wb_.append(data, datalen);
1791
1792 if (wb_.rleft() >= MAX_BUFFER_SIZE) {
1793 break;
1794 }
1795 }
1796
1797 if (nghttp2_session_want_read(session_) == 0 &&
1798 nghttp2_session_want_write(session_) == 0 && wb_.rleft() == 0) {
1799 if (LOG_ENABLED(INFO)) {
1800 SSLOG(INFO, this) << "No more read/write for this session";
1801 }
1802 return -1;
1803 }
1804
1805 return 0;
1806 }
1807
signal_write()1808 void Http2Session::signal_write() {
1809 switch (state_) {
1810 case Http2SessionState::DISCONNECTED:
1811 if (!ev_is_active(&initiate_connection_timer_)) {
1812 if (LOG_ENABLED(INFO)) {
1813 LOG(INFO) << "Start connecting to backend server";
1814 }
1815 // Since the timer is set to 0., these will feed 2 events. We
1816 // will stop the timer in the initiate_connection_timer_ to void
1817 // 2nd event.
1818 ev_timer_start(conn_.loop, &initiate_connection_timer_);
1819 ev_feed_event(conn_.loop, &initiate_connection_timer_, 0);
1820 }
1821 break;
1822 case Http2SessionState::CONNECTED:
1823 conn_.wlimit.startw();
1824 break;
1825 default:
1826 break;
1827 }
1828 }
1829
get_loop() const1830 struct ev_loop *Http2Session::get_loop() const { return conn_.loop; }
1831
get_wev()1832 ev_io *Http2Session::get_wev() { return &conn_.wev; }
1833
get_state() const1834 Http2SessionState Http2Session::get_state() const { return state_; }
1835
set_state(Http2SessionState state)1836 void Http2Session::set_state(Http2SessionState state) { state_ = state; }
1837
terminate_session(uint32_t error_code)1838 int Http2Session::terminate_session(uint32_t error_code) {
1839 int rv;
1840 rv = nghttp2_session_terminate_session(session_, error_code);
1841 if (rv != 0) {
1842 return -1;
1843 }
1844 return 0;
1845 }
1846
get_ssl() const1847 SSL *Http2Session::get_ssl() const { return conn_.tls.ssl; }
1848
consume(int32_t stream_id,size_t len)1849 int Http2Session::consume(int32_t stream_id, size_t len) {
1850 int rv;
1851
1852 if (!session_) {
1853 return 0;
1854 }
1855
1856 rv = nghttp2_session_consume(session_, stream_id, len);
1857
1858 if (rv != 0) {
1859 SSLOG(WARN, this) << "nghttp2_session_consume() returned error: "
1860 << nghttp2_strerror(rv);
1861
1862 return -1;
1863 }
1864
1865 return 0;
1866 }
1867
can_push_request(const Downstream * downstream) const1868 bool Http2Session::can_push_request(const Downstream *downstream) const {
1869 auto &req = downstream->request();
1870 return state_ == Http2SessionState::CONNECTED &&
1871 connection_check_state_ == ConnectionCheck::NONE &&
1872 (req.connect_proto == ConnectProto::NONE || settings_recved_);
1873 }
1874
start_checking_connection()1875 void Http2Session::start_checking_connection() {
1876 if (state_ != Http2SessionState::CONNECTED ||
1877 connection_check_state_ != ConnectionCheck::REQUIRED) {
1878 return;
1879 }
1880 connection_check_state_ = ConnectionCheck::STARTED;
1881
1882 SSLOG(INFO, this) << "Start checking connection";
1883 // If connection is down, we may get error when writing data. Issue
1884 // ping frame to see whether connection is alive.
1885 nghttp2_submit_ping(session_, NGHTTP2_FLAG_NONE, nullptr);
1886
1887 // set ping timeout and start timer again
1888 reset_connection_check_timer(CONNCHK_PING_TIMEOUT);
1889
1890 signal_write();
1891 }
1892
reset_connection_check_timer(ev_tstamp t)1893 void Http2Session::reset_connection_check_timer(ev_tstamp t) {
1894 connchk_timer_.repeat = t;
1895 ev_timer_again(conn_.loop, &connchk_timer_);
1896 }
1897
reset_connection_check_timer_if_not_checking()1898 void Http2Session::reset_connection_check_timer_if_not_checking() {
1899 if (connection_check_state_ != ConnectionCheck::NONE) {
1900 return;
1901 }
1902
1903 reset_connection_check_timer(CONNCHK_TIMEOUT);
1904 }
1905
connection_alive()1906 void Http2Session::connection_alive() {
1907 reset_connection_check_timer(CONNCHK_TIMEOUT);
1908
1909 if (connection_check_state_ == ConnectionCheck::NONE) {
1910 return;
1911 }
1912
1913 if (LOG_ENABLED(INFO)) {
1914 SSLOG(INFO, this) << "Connection alive";
1915 }
1916
1917 connection_check_state_ = ConnectionCheck::NONE;
1918
1919 submit_pending_requests();
1920 }
1921
submit_pending_requests()1922 void Http2Session::submit_pending_requests() {
1923 for (auto dconn = dconns_.head; dconn; dconn = dconn->dlnext) {
1924 auto downstream = dconn->get_downstream();
1925
1926 if (!downstream->get_request_pending() ||
1927 !downstream->request_submission_ready()) {
1928 continue;
1929 }
1930
1931 auto &req = downstream->request();
1932 if (req.connect_proto != ConnectProto::NONE && !settings_recved_) {
1933 continue;
1934 }
1935
1936 auto upstream = downstream->get_upstream();
1937
1938 if (dconn->push_request_headers() != 0) {
1939 if (LOG_ENABLED(INFO)) {
1940 SSLOG(INFO, this) << "backend request failed";
1941 }
1942
1943 upstream->on_downstream_abort_request(downstream, 400);
1944
1945 continue;
1946 }
1947
1948 upstream->resume_read(SHRPX_NO_BUFFER, downstream, 0);
1949 }
1950 }
1951
set_connection_check_state(ConnectionCheck state)1952 void Http2Session::set_connection_check_state(ConnectionCheck state) {
1953 connection_check_state_ = state;
1954 }
1955
get_connection_check_state() const1956 ConnectionCheck Http2Session::get_connection_check_state() const {
1957 return connection_check_state_;
1958 }
1959
noop()1960 int Http2Session::noop() { return 0; }
1961
read_noop(const uint8_t * data,size_t datalen)1962 int Http2Session::read_noop(const uint8_t *data, size_t datalen) { return 0; }
1963
write_noop()1964 int Http2Session::write_noop() { return 0; }
1965
connected()1966 int Http2Session::connected() {
1967 auto sock_error = util::get_socket_error(conn_.fd);
1968 if (sock_error != 0) {
1969 SSLOG(WARN, this) << "Backend connect failed; addr="
1970 << util::to_numeric_addr(raddr_)
1971 << ": errno=" << sock_error;
1972
1973 downstream_failure(addr_, raddr_);
1974
1975 return -1;
1976 }
1977
1978 if (LOG_ENABLED(INFO)) {
1979 SSLOG(INFO, this) << "Connection established";
1980 }
1981
1982 // Reset timeout for write. Previously, we set timeout for connect.
1983 conn_.wt.repeat = group_->shared_addr->timeout.write;
1984 ev_timer_again(conn_.loop, &conn_.wt);
1985
1986 conn_.rlimit.startw();
1987 conn_.again_rt();
1988
1989 read_ = &Http2Session::read_clear;
1990 write_ = &Http2Session::write_clear;
1991
1992 if (state_ == Http2SessionState::PROXY_CONNECTING) {
1993 return do_write();
1994 }
1995
1996 if (conn_.tls.ssl) {
1997 read_ = &Http2Session::tls_handshake;
1998 write_ = &Http2Session::tls_handshake;
1999
2000 return do_write();
2001 }
2002
2003 if (connection_made() != 0) {
2004 state_ = Http2SessionState::CONNECT_FAILING;
2005 return -1;
2006 }
2007
2008 return 0;
2009 }
2010
read_clear()2011 int Http2Session::read_clear() {
2012 conn_.last_read = std::chrono::steady_clock::now();
2013
2014 std::array<uint8_t, 16_k> buf;
2015
2016 for (;;) {
2017 auto nread = conn_.read_clear(buf.data(), buf.size());
2018
2019 if (nread == 0) {
2020 return write_clear();
2021 }
2022
2023 if (nread < 0) {
2024 return nread;
2025 }
2026
2027 if (on_read(buf.data(), nread) != 0) {
2028 return -1;
2029 }
2030 }
2031 }
2032
write_clear()2033 int Http2Session::write_clear() {
2034 conn_.last_read = std::chrono::steady_clock::now();
2035
2036 std::array<struct iovec, MAX_WR_IOVCNT> iov;
2037
2038 for (;;) {
2039 if (wb_.rleft() > 0) {
2040 auto iovcnt = wb_.riovec(iov.data(), iov.size());
2041 auto nwrite = conn_.writev_clear(iov.data(), iovcnt);
2042
2043 if (nwrite == 0) {
2044 return 0;
2045 }
2046
2047 if (nwrite < 0) {
2048 // We may have pending data in receive buffer which may
2049 // contain part of response body. So keep reading. Invoke
2050 // read event to get read(2) error just in case.
2051 ev_feed_event(conn_.loop, &conn_.rev, EV_READ);
2052 write_ = &Http2Session::write_void;
2053 break;
2054 }
2055
2056 wb_.drain(nwrite);
2057 continue;
2058 }
2059
2060 if (on_write() != 0) {
2061 return -1;
2062 }
2063 if (wb_.rleft() == 0) {
2064 break;
2065 }
2066 }
2067
2068 conn_.wlimit.stopw();
2069 ev_timer_stop(conn_.loop, &conn_.wt);
2070
2071 return 0;
2072 }
2073
tls_handshake()2074 int Http2Session::tls_handshake() {
2075 conn_.last_read = std::chrono::steady_clock::now();
2076
2077 ERR_clear_error();
2078
2079 auto rv = conn_.tls_handshake();
2080
2081 if (rv == SHRPX_ERR_INPROGRESS) {
2082 return 0;
2083 }
2084
2085 if (rv < 0) {
2086 downstream_failure(addr_, raddr_);
2087
2088 return rv;
2089 }
2090
2091 if (LOG_ENABLED(INFO)) {
2092 SSLOG(INFO, this) << "SSL/TLS handshake completed";
2093 }
2094
2095 if (!get_config()->tls.insecure &&
2096 tls::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) {
2097 downstream_failure(addr_, raddr_);
2098
2099 return -1;
2100 }
2101
2102 read_ = &Http2Session::read_tls;
2103 write_ = &Http2Session::write_tls;
2104
2105 if (connection_made() != 0) {
2106 state_ = Http2SessionState::CONNECT_FAILING;
2107 return -1;
2108 }
2109
2110 return 0;
2111 }
2112
read_tls()2113 int Http2Session::read_tls() {
2114 conn_.last_read = std::chrono::steady_clock::now();
2115
2116 std::array<uint8_t, 16_k> buf;
2117
2118 ERR_clear_error();
2119
2120 for (;;) {
2121 auto nread = conn_.read_tls(buf.data(), buf.size());
2122
2123 if (nread == 0) {
2124 return write_tls();
2125 }
2126
2127 if (nread < 0) {
2128 return nread;
2129 }
2130
2131 if (on_read(buf.data(), nread) != 0) {
2132 return -1;
2133 }
2134 }
2135 }
2136
write_tls()2137 int Http2Session::write_tls() {
2138 conn_.last_read = std::chrono::steady_clock::now();
2139
2140 ERR_clear_error();
2141
2142 struct iovec iov;
2143
2144 for (;;) {
2145 if (wb_.rleft() > 0) {
2146 auto iovcnt = wb_.riovec(&iov, 1);
2147 if (iovcnt != 1) {
2148 assert(0);
2149 return -1;
2150 }
2151 auto nwrite = conn_.write_tls(iov.iov_base, iov.iov_len);
2152
2153 if (nwrite == 0) {
2154 return 0;
2155 }
2156
2157 if (nwrite < 0) {
2158 // We may have pending data in receive buffer which may
2159 // contain part of response body. So keep reading. Invoke
2160 // read event to get read(2) error just in case.
2161 ev_feed_event(conn_.loop, &conn_.rev, EV_READ);
2162 write_ = &Http2Session::write_void;
2163 break;
2164 }
2165
2166 wb_.drain(nwrite);
2167
2168 continue;
2169 }
2170
2171 if (on_write() != 0) {
2172 return -1;
2173 }
2174 if (wb_.rleft() == 0) {
2175 conn_.start_tls_write_idle();
2176 break;
2177 }
2178 }
2179
2180 conn_.wlimit.stopw();
2181 ev_timer_stop(conn_.loop, &conn_.wt);
2182
2183 return 0;
2184 }
2185
write_void()2186 int Http2Session::write_void() {
2187 conn_.wlimit.stopw();
2188 return 0;
2189 }
2190
should_hard_fail() const2191 bool Http2Session::should_hard_fail() const {
2192 switch (state_) {
2193 case Http2SessionState::PROXY_CONNECTING:
2194 case Http2SessionState::PROXY_FAILED:
2195 return true;
2196 case Http2SessionState::DISCONNECTED: {
2197 const auto &proxy = get_config()->downstream_http_proxy;
2198 return !proxy.host.empty();
2199 }
2200 default:
2201 return false;
2202 }
2203 }
2204
get_addr() const2205 DownstreamAddr *Http2Session::get_addr() const { return addr_; }
2206
handle_downstream_push_promise(Downstream * downstream,int32_t promised_stream_id)2207 int Http2Session::handle_downstream_push_promise(Downstream *downstream,
2208 int32_t promised_stream_id) {
2209 auto upstream = downstream->get_upstream();
2210 if (!upstream->push_enabled()) {
2211 return -1;
2212 }
2213
2214 auto promised_downstream =
2215 upstream->on_downstream_push_promise(downstream, promised_stream_id);
2216 if (!promised_downstream) {
2217 return -1;
2218 }
2219
2220 // Now we have Downstream object for pushed stream.
2221 // promised_downstream->get_stream() still returns 0.
2222
2223 auto handler = upstream->get_client_handler();
2224
2225 auto promised_dconn = std::make_unique<Http2DownstreamConnection>(this);
2226 promised_dconn->set_client_handler(handler);
2227
2228 auto ptr = promised_dconn.get();
2229
2230 if (promised_downstream->attach_downstream_connection(
2231 std::move(promised_dconn)) != 0) {
2232 return -1;
2233 }
2234
2235 auto promised_sd = std::make_unique<StreamData>();
2236
2237 nghttp2_session_set_stream_user_data(session_, promised_stream_id,
2238 promised_sd.get());
2239
2240 ptr->attach_stream_data(promised_sd.get());
2241 streams_.append(promised_sd.release());
2242
2243 return 0;
2244 }
2245
handle_downstream_push_promise_complete(Downstream * downstream,Downstream * promised_downstream)2246 int Http2Session::handle_downstream_push_promise_complete(
2247 Downstream *downstream, Downstream *promised_downstream) {
2248 auto &promised_req = promised_downstream->request();
2249
2250 auto &promised_balloc = promised_downstream->get_block_allocator();
2251
2252 auto authority = promised_req.fs.header(http2::HD__AUTHORITY);
2253 auto path = promised_req.fs.header(http2::HD__PATH);
2254 auto method = promised_req.fs.header(http2::HD__METHOD);
2255 auto scheme = promised_req.fs.header(http2::HD__SCHEME);
2256
2257 if (!authority) {
2258 authority = promised_req.fs.header(http2::HD_HOST);
2259 }
2260
2261 auto method_token = http2::lookup_method_token(method->value);
2262 if (method_token == -1) {
2263 if (LOG_ENABLED(INFO)) {
2264 SSLOG(INFO, this) << "Unrecognized method: " << method->value;
2265 }
2266
2267 return -1;
2268 }
2269
2270 // TODO Rewrite authority if we enabled rewrite host. But we
2271 // really don't know how to rewrite host. Should we use the same
2272 // host in associated stream?
2273 if (authority) {
2274 promised_req.authority = authority->value;
2275 }
2276 promised_req.method = method_token;
2277 // libnghttp2 ensures that we don't have CONNECT method in
2278 // PUSH_PROMISE, and guarantees that :scheme exists.
2279 if (scheme) {
2280 promised_req.scheme = scheme->value;
2281 }
2282
2283 // For server-wide OPTIONS request, path is empty.
2284 if (method_token != HTTP_OPTIONS || path->value != "*"_sr) {
2285 promised_req.path = http2::rewrite_clean_path(promised_balloc, path->value);
2286 }
2287
2288 promised_downstream->inspect_http2_request();
2289
2290 auto upstream = promised_downstream->get_upstream();
2291
2292 promised_downstream->set_request_state(DownstreamState::MSG_COMPLETE);
2293 promised_downstream->set_request_header_sent(true);
2294
2295 if (upstream->on_downstream_push_promise_complete(downstream,
2296 promised_downstream) != 0) {
2297 return -1;
2298 }
2299
2300 return 0;
2301 }
2302
get_num_dconns() const2303 size_t Http2Session::get_num_dconns() const { return dconns_.size(); }
2304
max_concurrency_reached(size_t extra) const2305 bool Http2Session::max_concurrency_reached(size_t extra) const {
2306 if (!session_) {
2307 return dconns_.size() + extra >= 100;
2308 }
2309
2310 // If session does not allow further requests, it effectively means
2311 // that maximum concurrency is reached.
2312 return !nghttp2_session_check_request_allowed(session_) ||
2313 dconns_.size() + extra >=
2314 nghttp2_session_get_remote_settings(
2315 session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS);
2316 }
2317
2318 const std::shared_ptr<DownstreamAddrGroup> &
get_downstream_addr_group() const2319 Http2Session::get_downstream_addr_group() const {
2320 return group_;
2321 }
2322
add_to_extra_freelist()2323 void Http2Session::add_to_extra_freelist() {
2324 if (freelist_zone_ != FreelistZone::NONE) {
2325 return;
2326 }
2327
2328 if (LOG_ENABLED(INFO)) {
2329 SSLOG(INFO, this) << "Append to http2_extra_freelist, addr=" << addr_
2330 << ", freelist.size="
2331 << addr_->http2_extra_freelist.size();
2332 }
2333
2334 freelist_zone_ = FreelistZone::EXTRA;
2335 addr_->http2_extra_freelist.append(this);
2336 }
2337
remove_from_freelist()2338 void Http2Session::remove_from_freelist() {
2339 switch (freelist_zone_) {
2340 case FreelistZone::NONE:
2341 return;
2342 case FreelistZone::EXTRA:
2343 if (LOG_ENABLED(INFO)) {
2344 SSLOG(INFO, this) << "Remove from http2_extra_freelist, addr=" << addr_
2345 << ", freelist.size="
2346 << addr_->http2_extra_freelist.size();
2347 }
2348 addr_->http2_extra_freelist.remove(this);
2349 break;
2350 case FreelistZone::GONE:
2351 return;
2352 }
2353
2354 freelist_zone_ = FreelistZone::NONE;
2355 }
2356
exclude_from_scheduling()2357 void Http2Session::exclude_from_scheduling() {
2358 remove_from_freelist();
2359 freelist_zone_ = FreelistZone::GONE;
2360 }
2361
get_request_buf()2362 DefaultMemchunks *Http2Session::get_request_buf() { return &wb_; }
2363
on_timeout()2364 void Http2Session::on_timeout() {
2365 switch (state_) {
2366 case Http2SessionState::PROXY_CONNECTING: {
2367 auto worker_blocker = worker_->get_connect_blocker();
2368 worker_blocker->on_failure();
2369 break;
2370 }
2371 case Http2SessionState::CONNECTING:
2372 SSLOG(WARN, this) << "Connect time out; addr="
2373 << util::to_numeric_addr(raddr_);
2374
2375 downstream_failure(addr_, raddr_);
2376 break;
2377 default:
2378 break;
2379 }
2380 }
2381
check_retire()2382 void Http2Session::check_retire() {
2383 if (!group_->retired) {
2384 return;
2385 }
2386
2387 ev_prepare_stop(conn_.loop, &prep_);
2388
2389 if (!session_) {
2390 return;
2391 }
2392
2393 auto last_stream_id = nghttp2_session_get_last_proc_stream_id(session_);
2394 nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, last_stream_id,
2395 NGHTTP2_NO_ERROR, nullptr, 0);
2396
2397 signal_write();
2398 }
2399
get_raddr() const2400 const Address *Http2Session::get_raddr() const { return raddr_; }
2401
on_settings_received(const nghttp2_frame * frame)2402 void Http2Session::on_settings_received(const nghttp2_frame *frame) {
2403 // TODO This effectively disallows nghttpx to change its behaviour
2404 // based on the 2nd SETTINGS.
2405 if (settings_recved_) {
2406 return;
2407 }
2408
2409 settings_recved_ = true;
2410
2411 for (size_t i = 0; i < frame->settings.niv; ++i) {
2412 auto &ent = frame->settings.iv[i];
2413 if (ent.settings_id == NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL) {
2414 allow_connect_proto_ = true;
2415 break;
2416 }
2417 }
2418
2419 submit_pending_requests();
2420 }
2421
get_allow_connect_proto() const2422 bool Http2Session::get_allow_connect_proto() const {
2423 return allow_connect_proto_;
2424 }
2425
2426 } // namespace shrpx
2427