1 /*
2 * nghttp2 - HTTP/2 C Library
3 *
4 * Copyright (c) 2012 Tatsuhiro Tsujikawa
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 */
25 #include "shrpx_http2_session.h"
26
27 #include <netinet/tcp.h>
28 #ifdef HAVE_UNISTD_H
29 # include <unistd.h>
30 #endif // HAVE_UNISTD_H
31
32 #include <vector>
33
34 #include <openssl/err.h>
35
36 #include "shrpx_upstream.h"
37 #include "shrpx_downstream.h"
38 #include "shrpx_config.h"
39 #include "shrpx_error.h"
40 #include "shrpx_http2_downstream_connection.h"
41 #include "shrpx_client_handler.h"
42 #include "shrpx_tls.h"
43 #include "shrpx_http.h"
44 #include "shrpx_worker.h"
45 #include "shrpx_connect_blocker.h"
46 #include "shrpx_log.h"
47 #include "http2.h"
48 #include "util.h"
49 #include "base64.h"
50 #include "tls.h"
51
52 using namespace nghttp2;
53
54 namespace shrpx {
55
56 namespace {
57 constexpr ev_tstamp CONNCHK_TIMEOUT = 5.;
58 constexpr ev_tstamp CONNCHK_PING_TIMEOUT = 1.;
59 } // namespace
60
61 namespace {
62 constexpr size_t MAX_BUFFER_SIZE = 32_k;
63 } // namespace
64
65 namespace {
connchk_timeout_cb(struct ev_loop * loop,ev_timer * w,int revents)66 void connchk_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
67 auto http2session = static_cast<Http2Session *>(w->data);
68
69 ev_timer_stop(loop, w);
70
71 switch (http2session->get_connection_check_state()) {
72 case ConnectionCheck::STARTED:
73 // ping timeout; disconnect
74 if (LOG_ENABLED(INFO)) {
75 SSLOG(INFO, http2session) << "ping timeout";
76 }
77
78 delete http2session;
79
80 return;
81 default:
82 if (LOG_ENABLED(INFO)) {
83 SSLOG(INFO, http2session) << "connection check required";
84 }
85 http2session->set_connection_check_state(ConnectionCheck::REQUIRED);
86 }
87 }
88 } // namespace
89
90 namespace {
settings_timeout_cb(struct ev_loop * loop,ev_timer * w,int revents)91 void settings_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
92 auto http2session = static_cast<Http2Session *>(w->data);
93
94 if (LOG_ENABLED(INFO)) {
95 SSLOG(INFO, http2session) << "SETTINGS timeout";
96 }
97
98 downstream_failure(http2session->get_addr(), http2session->get_raddr());
99
100 if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) {
101 delete http2session;
102
103 return;
104 }
105 http2session->signal_write();
106 }
107 } // namespace
108
109 namespace {
timeoutcb(struct ev_loop * loop,ev_timer * w,int revents)110 void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
111 auto conn = static_cast<Connection *>(w->data);
112 auto http2session = static_cast<Http2Session *>(conn->data);
113
114 if (w == &conn->rt && !conn->expired_rt()) {
115 return;
116 }
117
118 if (LOG_ENABLED(INFO)) {
119 SSLOG(INFO, http2session) << "Timeout";
120 }
121
122 http2session->on_timeout();
123
124 delete http2session;
125 }
126 } // namespace
127
128 namespace {
readcb(struct ev_loop * loop,ev_io * w,int revents)129 void readcb(struct ev_loop *loop, ev_io *w, int revents) {
130 int rv;
131 auto conn = static_cast<Connection *>(w->data);
132 auto http2session = static_cast<Http2Session *>(conn->data);
133 rv = http2session->do_read();
134 if (rv != 0) {
135 delete http2session;
136
137 return;
138 }
139 http2session->connection_alive();
140 }
141 } // namespace
142
143 namespace {
writecb(struct ev_loop * loop,ev_io * w,int revents)144 void writecb(struct ev_loop *loop, ev_io *w, int revents) {
145 int rv;
146 auto conn = static_cast<Connection *>(w->data);
147 auto http2session = static_cast<Http2Session *>(conn->data);
148 rv = http2session->do_write();
149 if (rv != 0) {
150 delete http2session;
151
152 return;
153 }
154 http2session->reset_connection_check_timer_if_not_checking();
155 }
156 } // namespace
157
158 namespace {
initiate_connection_cb(struct ev_loop * loop,ev_timer * w,int revents)159 void initiate_connection_cb(struct ev_loop *loop, ev_timer *w, int revents) {
160 auto http2session = static_cast<Http2Session *>(w->data);
161 ev_timer_stop(loop, w);
162 if (http2session->initiate_connection() != 0) {
163 if (LOG_ENABLED(INFO)) {
164 SSLOG(INFO, http2session) << "Could not initiate backend connection";
165 }
166
167 delete http2session;
168
169 return;
170 }
171 }
172 } // namespace
173
174 namespace {
prepare_cb(struct ev_loop * loop,ev_prepare * w,int revents)175 void prepare_cb(struct ev_loop *loop, ev_prepare *w, int revents) {
176 auto http2session = static_cast<Http2Session *>(w->data);
177 http2session->check_retire();
178 }
179 } // namespace
180
Http2Session(struct ev_loop * loop,SSL_CTX * ssl_ctx,Worker * worker,const std::shared_ptr<DownstreamAddrGroup> & group,DownstreamAddr * addr)181 Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
182 Worker *worker,
183 const std::shared_ptr<DownstreamAddrGroup> &group,
184 DownstreamAddr *addr)
185 : dlnext(nullptr),
186 dlprev(nullptr),
187 conn_(loop, -1, nullptr, worker->get_mcpool(),
188 group->shared_addr->timeout.write, group->shared_addr->timeout.read,
189 {}, {}, writecb, readcb, timeoutcb, this,
190 get_config()->tls.dyn_rec.warmup_threshold,
191 get_config()->tls.dyn_rec.idle_timeout, Proto::HTTP2),
192 wb_(worker->get_mcpool()),
193 worker_(worker),
194 ssl_ctx_(ssl_ctx),
195 group_(group),
196 addr_(addr),
197 session_(nullptr),
198 raddr_(nullptr),
199 state_(Http2SessionState::DISCONNECTED),
200 connection_check_state_(ConnectionCheck::NONE),
201 freelist_zone_(FreelistZone::NONE),
202 settings_recved_(false),
203 allow_connect_proto_(false) {
204 read_ = write_ = &Http2Session::noop;
205
206 on_read_ = &Http2Session::read_noop;
207 on_write_ = &Http2Session::write_noop;
208
209 // We will reuse this many times, so use repeat timeout value. The
210 // timeout value is set later.
211 ev_timer_init(&connchk_timer_, connchk_timeout_cb, 0., 0.);
212
213 connchk_timer_.data = this;
214
215 // SETTINGS ACK timeout is 10 seconds for now. We will reuse this
216 // many times, so use repeat timeout value.
217 ev_timer_init(&settings_timer_, settings_timeout_cb, 0., 0.);
218
219 settings_timer_.data = this;
220
221 ev_timer_init(&initiate_connection_timer_, initiate_connection_cb, 0., 0.);
222 initiate_connection_timer_.data = this;
223
224 ev_prepare_init(&prep_, prepare_cb);
225 prep_.data = this;
226 ev_prepare_start(loop, &prep_);
227 }
228
~Http2Session()229 Http2Session::~Http2Session() {
230 exclude_from_scheduling();
231 disconnect(should_hard_fail());
232 }
233
disconnect(bool hard)234 int Http2Session::disconnect(bool hard) {
235 if (LOG_ENABLED(INFO)) {
236 SSLOG(INFO, this) << "Disconnecting";
237 }
238 nghttp2_session_del(session_);
239 session_ = nullptr;
240
241 wb_.reset();
242
243 if (dns_query_) {
244 auto dns_tracker = worker_->get_dns_tracker();
245 dns_tracker->cancel(dns_query_.get());
246 }
247
248 conn_.rlimit.stopw();
249 conn_.wlimit.stopw();
250
251 ev_prepare_stop(conn_.loop, &prep_);
252
253 ev_timer_stop(conn_.loop, &initiate_connection_timer_);
254 ev_timer_stop(conn_.loop, &settings_timer_);
255 ev_timer_stop(conn_.loop, &connchk_timer_);
256
257 read_ = write_ = &Http2Session::noop;
258
259 on_read_ = &Http2Session::read_noop;
260 on_write_ = &Http2Session::write_noop;
261
262 conn_.disconnect();
263
264 if (proxy_htp_) {
265 proxy_htp_.reset();
266 }
267
268 connection_check_state_ = ConnectionCheck::NONE;
269 state_ = Http2SessionState::DISCONNECTED;
270
271 // When deleting Http2DownstreamConnection, it calls this object's
272 // remove_downstream_connection(). The multiple
273 // Http2DownstreamConnection objects belong to the same
274 // ClientHandler object if upstream is h2. So be careful when you
275 // delete ClientHandler here.
276 //
277 // We allow creating new pending Http2DownstreamConnection with this
278 // object. Upstream::on_downstream_reset() may add
279 // Http2DownstreamConnection to another Http2Session.
280
281 for (auto dc = dconns_.head; dc;) {
282 auto next = dc->dlnext;
283 auto downstream = dc->get_downstream();
284 auto upstream = downstream->get_upstream();
285
286 // Failure is allowed only for HTTP/1 upstream where upstream is
287 // not shared by multiple Downstreams.
288 if (upstream->on_downstream_reset(downstream, hard) != 0) {
289 delete upstream->get_client_handler();
290 }
291
292 // dc was deleted
293 dc = next;
294 }
295
296 auto streams = std::move(streams_);
297 for (auto s = streams.head; s;) {
298 auto next = s->dlnext;
299 delete s;
300 s = next;
301 }
302
303 return 0;
304 }
305
resolve_name()306 int Http2Session::resolve_name() {
307 auto dns_query = std::make_unique<DNSQuery>(
308 addr_->host, [this](DNSResolverStatus status, const Address *result) {
309 int rv;
310
311 if (status == DNSResolverStatus::OK) {
312 *resolved_addr_ = *result;
313 util::set_port(*this->resolved_addr_, this->addr_->port);
314 }
315
316 rv = this->initiate_connection();
317 if (rv != 0) {
318 delete this;
319 }
320 });
321 resolved_addr_ = std::make_unique<Address>();
322 auto dns_tracker = worker_->get_dns_tracker();
323 switch (dns_tracker->resolve(resolved_addr_.get(), dns_query.get())) {
324 case DNSResolverStatus::ERROR:
325 return -1;
326 case DNSResolverStatus::RUNNING:
327 dns_query_ = std::move(dns_query);
328 state_ = Http2SessionState::RESOLVING_NAME;
329 return 0;
330 case DNSResolverStatus::OK:
331 util::set_port(*resolved_addr_, addr_->port);
332 return 0;
333 default:
334 assert(0);
335 abort();
336 }
337 }
338
339 namespace {
340 int htp_hdrs_completecb(llhttp_t *htp);
341 } // namespace
342
343 namespace {
344 constexpr llhttp_settings_t htp_hooks = {
345 nullptr, // llhttp_cb on_message_begin;
346 nullptr, // llhttp_data_cb on_url;
347 nullptr, // llhttp_data_cb on_status;
348 nullptr, // llhttp_data_cb on_method;
349 nullptr, // llhttp_data_cb on_version;
350 nullptr, // llhttp_data_cb on_header_field;
351 nullptr, // llhttp_data_cb on_header_value;
352 nullptr, // llhttp_data_cb on_chunk_extension_name;
353 nullptr, // llhttp_data_cb on_chunk_extension_value;
354 htp_hdrs_completecb, // llhttp_cb on_headers_complete;
355 nullptr, // llhttp_data_cb on_body;
356 nullptr, // llhttp_cb on_message_complete;
357 nullptr, // llhttp_cb on_url_complete;
358 nullptr, // llhttp_cb on_status_complete;
359 nullptr, // llhttp_cb on_method_complete;
360 nullptr, // llhttp_cb on_version_complete;
361 nullptr, // llhttp_cb on_header_field_complete;
362 nullptr, // llhttp_cb on_header_value_complete;
363 nullptr, // llhttp_cb on_chunk_extension_name_complete;
364 nullptr, // llhttp_cb on_chunk_extension_value_complete;
365 nullptr, // llhttp_cb on_chunk_header;
366 nullptr, // llhttp_cb on_chunk_complete;
367 nullptr, // llhttp_cb on_reset;
368 };
369 } // namespace
370
initiate_connection()371 int Http2Session::initiate_connection() {
372 int rv = 0;
373
374 auto worker_blocker = worker_->get_connect_blocker();
375
376 if (state_ == Http2SessionState::DISCONNECTED ||
377 state_ == Http2SessionState::RESOLVING_NAME) {
378 if (worker_blocker->blocked()) {
379 if (LOG_ENABLED(INFO)) {
380 SSLOG(INFO, this)
381 << "Worker wide backend connection was blocked temporarily";
382 }
383 return -1;
384 }
385 }
386
387 auto &downstreamconf = *get_config()->conn.downstream;
388
389 const auto &proxy = get_config()->downstream_http_proxy;
390 if (!proxy.host.empty() && state_ == Http2SessionState::DISCONNECTED) {
391 if (LOG_ENABLED(INFO)) {
392 SSLOG(INFO, this) << "Connecting to the proxy " << proxy.host << ":"
393 << proxy.port;
394 }
395
396 conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family);
397
398 if (conn_.fd == -1) {
399 auto error = errno;
400 SSLOG(WARN, this) << "Backend proxy socket() failed; addr="
401 << util::to_numeric_addr(&proxy.addr)
402 << ", errno=" << error;
403
404 worker_blocker->on_failure();
405 return -1;
406 }
407
408 rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len);
409 if (rv != 0 && errno != EINPROGRESS) {
410 auto error = errno;
411 SSLOG(WARN, this) << "Backend proxy connect() failed; addr="
412 << util::to_numeric_addr(&proxy.addr)
413 << ", errno=" << error;
414
415 worker_blocker->on_failure();
416
417 return -1;
418 }
419
420 raddr_ = &proxy.addr;
421
422 worker_blocker->on_success();
423
424 ev_io_set(&conn_.rev, conn_.fd, EV_READ);
425 ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
426
427 conn_.wlimit.startw();
428
429 conn_.wt.repeat = downstreamconf.timeout.connect;
430 ev_timer_again(conn_.loop, &conn_.wt);
431
432 write_ = &Http2Session::connected;
433
434 on_read_ = &Http2Session::downstream_read_proxy;
435 on_write_ = &Http2Session::downstream_connect_proxy;
436
437 proxy_htp_ = std::make_unique<llhttp_t>();
438 llhttp_init(proxy_htp_.get(), HTTP_RESPONSE, &htp_hooks);
439 proxy_htp_->data = this;
440
441 state_ = Http2SessionState::PROXY_CONNECTING;
442
443 return 0;
444 }
445
446 if (state_ == Http2SessionState::DISCONNECTED ||
447 state_ == Http2SessionState::PROXY_CONNECTED ||
448 state_ == Http2SessionState::RESOLVING_NAME) {
449 if (LOG_ENABLED(INFO)) {
450 if (state_ != Http2SessionState::RESOLVING_NAME) {
451 SSLOG(INFO, this) << "Connecting to downstream server";
452 }
453 }
454 if (addr_->tls) {
455 assert(ssl_ctx_);
456
457 if (state_ != Http2SessionState::RESOLVING_NAME) {
458 auto ssl = tls::create_ssl(ssl_ctx_);
459 if (!ssl) {
460 return -1;
461 }
462
463 tls::setup_downstream_http2_alpn(ssl);
464
465 conn_.set_ssl(ssl);
466 conn_.tls.client_session_cache = &addr_->tls_session_cache;
467
468 auto sni_name =
469 addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni};
470
471 if (!util::numeric_host(sni_name.c_str())) {
472 // TLS extensions: SNI. There is no documentation about the return
473 // code for this function (actually this is macro wrapping SSL_ctrl
474 // at the time of this writing).
475 SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str());
476 }
477
478 auto tls_session = tls::reuse_tls_session(addr_->tls_session_cache);
479 if (tls_session) {
480 SSL_set_session(conn_.tls.ssl, tls_session);
481 SSL_SESSION_free(tls_session);
482 }
483 }
484
485 if (state_ == Http2SessionState::DISCONNECTED) {
486 if (addr_->dns) {
487 rv = resolve_name();
488 if (rv != 0) {
489 downstream_failure(addr_, nullptr);
490 return -1;
491 }
492 if (state_ == Http2SessionState::RESOLVING_NAME) {
493 return 0;
494 }
495 raddr_ = resolved_addr_.get();
496 } else {
497 raddr_ = &addr_->addr;
498 }
499 }
500
501 if (state_ == Http2SessionState::RESOLVING_NAME) {
502 if (dns_query_->status == DNSResolverStatus::ERROR) {
503 downstream_failure(addr_, nullptr);
504 return -1;
505 }
506 assert(dns_query_->status == DNSResolverStatus::OK);
507 state_ = Http2SessionState::DISCONNECTED;
508 dns_query_.reset();
509 raddr_ = resolved_addr_.get();
510 }
511
512 // If state_ == Http2SessionState::PROXY_CONNECTED, we have
513 // connected to the proxy using conn_.fd and tunnel has been
514 // established.
515 if (state_ == Http2SessionState::DISCONNECTED) {
516 assert(conn_.fd == -1);
517
518 conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
519 if (conn_.fd == -1) {
520 auto error = errno;
521 SSLOG(WARN, this)
522 << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
523 << ", errno=" << error;
524
525 worker_blocker->on_failure();
526 return -1;
527 }
528
529 worker_blocker->on_success();
530
531 rv = connect(conn_.fd,
532 // TODO maybe not thread-safe?
533 const_cast<sockaddr *>(&raddr_->su.sa), raddr_->len);
534 if (rv != 0 && errno != EINPROGRESS) {
535 auto error = errno;
536 SSLOG(WARN, this)
537 << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
538 << ", errno=" << error;
539
540 downstream_failure(addr_, raddr_);
541 return -1;
542 }
543
544 ev_io_set(&conn_.rev, conn_.fd, EV_READ);
545 ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
546 }
547
548 conn_.prepare_client_handshake();
549 } else {
550 if (state_ == Http2SessionState::DISCONNECTED) {
551 // Without TLS and proxy.
552 if (addr_->dns) {
553 rv = resolve_name();
554 if (rv != 0) {
555 downstream_failure(addr_, nullptr);
556 return -1;
557 }
558 if (state_ == Http2SessionState::RESOLVING_NAME) {
559 return 0;
560 }
561 raddr_ = resolved_addr_.get();
562 } else {
563 raddr_ = &addr_->addr;
564 }
565 }
566
567 if (state_ == Http2SessionState::RESOLVING_NAME) {
568 if (dns_query_->status == DNSResolverStatus::ERROR) {
569 downstream_failure(addr_, nullptr);
570 return -1;
571 }
572 assert(dns_query_->status == DNSResolverStatus::OK);
573 state_ = Http2SessionState::DISCONNECTED;
574 dns_query_.reset();
575 raddr_ = resolved_addr_.get();
576 }
577
578 if (state_ == Http2SessionState::DISCONNECTED) {
579 // Without TLS and proxy.
580 assert(conn_.fd == -1);
581
582 conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
583
584 if (conn_.fd == -1) {
585 auto error = errno;
586 SSLOG(WARN, this)
587 << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
588 << ", errno=" << error;
589
590 worker_blocker->on_failure();
591 return -1;
592 }
593
594 worker_blocker->on_success();
595
596 rv = connect(conn_.fd, const_cast<sockaddr *>(&raddr_->su.sa),
597 raddr_->len);
598 if (rv != 0 && errno != EINPROGRESS) {
599 auto error = errno;
600 SSLOG(WARN, this)
601 << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
602 << ", errno=" << error;
603
604 downstream_failure(addr_, raddr_);
605 return -1;
606 }
607
608 ev_io_set(&conn_.rev, conn_.fd, EV_READ);
609 ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
610 }
611 }
612
613 // We have been already connected when no TLS and proxy is used.
614 if (state_ == Http2SessionState::PROXY_CONNECTED) {
615 on_read_ = &Http2Session::read_noop;
616 on_write_ = &Http2Session::write_noop;
617
618 return connected();
619 }
620
621 write_ = &Http2Session::connected;
622
623 state_ = Http2SessionState::CONNECTING;
624 conn_.wlimit.startw();
625
626 conn_.wt.repeat = downstreamconf.timeout.connect;
627 ev_timer_again(conn_.loop, &conn_.wt);
628
629 return 0;
630 }
631
632 // Unreachable
633 assert(0);
634
635 return 0;
636 }
637
638 namespace {
htp_hdrs_completecb(llhttp_t * htp)639 int htp_hdrs_completecb(llhttp_t *htp) {
640 auto http2session = static_cast<Http2Session *>(htp->data);
641
642 // We only read HTTP header part. If tunneling succeeds, response
643 // body is a different protocol (HTTP/2 in this case), we don't read
644 // them here.
645
646 // We just check status code here
647 if (htp->status_code / 100 == 2) {
648 if (LOG_ENABLED(INFO)) {
649 SSLOG(INFO, http2session) << "Tunneling success";
650 }
651 http2session->set_state(Http2SessionState::PROXY_CONNECTED);
652
653 return HPE_PAUSED;
654 }
655
656 SSLOG(WARN, http2session) << "Tunneling failed: " << htp->status_code;
657 http2session->set_state(Http2SessionState::PROXY_FAILED);
658
659 return HPE_PAUSED;
660 }
661 } // namespace
662
downstream_read_proxy(const uint8_t * data,size_t datalen)663 int Http2Session::downstream_read_proxy(const uint8_t *data, size_t datalen) {
664 auto htperr = llhttp_execute(proxy_htp_.get(),
665 reinterpret_cast<const char *>(data), datalen);
666 if (htperr == HPE_PAUSED) {
667 switch (state_) {
668 case Http2SessionState::PROXY_CONNECTED:
669 // Initiate SSL/TLS handshake through established tunnel.
670 if (initiate_connection() != 0) {
671 return -1;
672 }
673 return 0;
674 case Http2SessionState::PROXY_FAILED:
675 return -1;
676 default:
677 break;
678 }
679 // should not be here
680 assert(0);
681 }
682
683 if (htperr != HPE_OK) {
684 return -1;
685 }
686
687 return 0;
688 }
689
downstream_connect_proxy()690 int Http2Session::downstream_connect_proxy() {
691 if (LOG_ENABLED(INFO)) {
692 SSLOG(INFO, this) << "Connected to the proxy";
693 }
694
695 std::string req = "CONNECT ";
696 req.append(addr_->hostport.c_str(), addr_->hostport.size());
697 if (addr_->port == 80 || addr_->port == 443) {
698 req += ':';
699 req += util::utos(addr_->port);
700 }
701 req += " HTTP/1.1\r\nHost: ";
702 req += addr_->host;
703 req += "\r\n";
704 const auto &proxy = get_config()->downstream_http_proxy;
705 if (!proxy.userinfo.empty()) {
706 req += "Proxy-Authorization: Basic ";
707 req += base64::encode(std::begin(proxy.userinfo), std::end(proxy.userinfo));
708 req += "\r\n";
709 }
710 req += "\r\n";
711 if (LOG_ENABLED(INFO)) {
712 SSLOG(INFO, this) << "HTTP proxy request headers\n" << req;
713 }
714 wb_.append(req);
715
716 on_write_ = &Http2Session::write_noop;
717
718 signal_write();
719 return 0;
720 }
721
add_downstream_connection(Http2DownstreamConnection * dconn)722 void Http2Session::add_downstream_connection(Http2DownstreamConnection *dconn) {
723 dconns_.append(dconn);
724 ++addr_->num_dconn;
725 }
726
remove_downstream_connection(Http2DownstreamConnection * dconn)727 void Http2Session::remove_downstream_connection(
728 Http2DownstreamConnection *dconn) {
729 --addr_->num_dconn;
730 dconns_.remove(dconn);
731 dconn->detach_stream_data();
732
733 if (LOG_ENABLED(INFO)) {
734 SSLOG(INFO, this) << "Remove downstream";
735 }
736
737 if (freelist_zone_ == FreelistZone::NONE && !max_concurrency_reached()) {
738 if (LOG_ENABLED(INFO)) {
739 SSLOG(INFO, this) << "Append to http2_extra_freelist, addr=" << addr_
740 << ", freelist.size="
741 << addr_->http2_extra_freelist.size();
742 }
743
744 add_to_extra_freelist();
745 }
746 }
747
remove_stream_data(StreamData * sd)748 void Http2Session::remove_stream_data(StreamData *sd) {
749 streams_.remove(sd);
750 if (sd->dconn) {
751 sd->dconn->detach_stream_data();
752 }
753 delete sd;
754 }
755
submit_request(Http2DownstreamConnection * dconn,const nghttp2_nv * nva,size_t nvlen,const nghttp2_data_provider * data_prd)756 int Http2Session::submit_request(Http2DownstreamConnection *dconn,
757 const nghttp2_nv *nva, size_t nvlen,
758 const nghttp2_data_provider *data_prd) {
759 assert(state_ == Http2SessionState::CONNECTED);
760 auto sd = std::make_unique<StreamData>();
761 sd->dlnext = sd->dlprev = nullptr;
762 // TODO Specify nullptr to pri_spec for now
763 auto stream_id =
764 nghttp2_submit_request(session_, nullptr, nva, nvlen, data_prd, sd.get());
765 if (stream_id < 0) {
766 SSLOG(FATAL, this) << "nghttp2_submit_request() failed: "
767 << nghttp2_strerror(stream_id);
768 return -1;
769 }
770
771 dconn->attach_stream_data(sd.get());
772 dconn->get_downstream()->set_downstream_stream_id(stream_id);
773 streams_.append(sd.release());
774
775 return 0;
776 }
777
submit_rst_stream(int32_t stream_id,uint32_t error_code)778 int Http2Session::submit_rst_stream(int32_t stream_id, uint32_t error_code) {
779 assert(state_ == Http2SessionState::CONNECTED);
780 if (LOG_ENABLED(INFO)) {
781 SSLOG(INFO, this) << "RST_STREAM stream_id=" << stream_id
782 << " with error_code=" << error_code;
783 }
784 int rv = nghttp2_submit_rst_stream(session_, NGHTTP2_FLAG_NONE, stream_id,
785 error_code);
786 if (rv != 0) {
787 SSLOG(FATAL, this) << "nghttp2_submit_rst_stream() failed: "
788 << nghttp2_strerror(rv);
789 return -1;
790 }
791 return 0;
792 }
793
get_session() const794 nghttp2_session *Http2Session::get_session() const { return session_; }
795
resume_data(Http2DownstreamConnection * dconn)796 int Http2Session::resume_data(Http2DownstreamConnection *dconn) {
797 assert(state_ == Http2SessionState::CONNECTED);
798 auto downstream = dconn->get_downstream();
799 int rv = nghttp2_session_resume_data(session_,
800 downstream->get_downstream_stream_id());
801 switch (rv) {
802 case 0:
803 case NGHTTP2_ERR_INVALID_ARGUMENT:
804 return 0;
805 default:
806 SSLOG(FATAL, this) << "nghttp2_resume_session() failed: "
807 << nghttp2_strerror(rv);
808 return -1;
809 }
810 }
811
812 namespace {
call_downstream_readcb(Http2Session * http2session,Downstream * downstream)813 void call_downstream_readcb(Http2Session *http2session,
814 Downstream *downstream) {
815 auto upstream = downstream->get_upstream();
816 if (!upstream) {
817 return;
818 }
819 if (upstream->downstream_read(downstream->get_downstream_connection()) != 0) {
820 delete upstream->get_client_handler();
821 }
822 }
823 } // namespace
824
825 namespace {
on_stream_close_callback(nghttp2_session * session,int32_t stream_id,uint32_t error_code,void * user_data)826 int on_stream_close_callback(nghttp2_session *session, int32_t stream_id,
827 uint32_t error_code, void *user_data) {
828 auto http2session = static_cast<Http2Session *>(user_data);
829 if (LOG_ENABLED(INFO)) {
830 SSLOG(INFO, http2session)
831 << "Stream stream_id=" << stream_id
832 << " is being closed with error code " << error_code;
833 }
834 auto sd = static_cast<StreamData *>(
835 nghttp2_session_get_stream_user_data(session, stream_id));
836 if (sd == 0) {
837 // We might get this close callback when pushed streams are
838 // closed.
839 return 0;
840 }
841 auto dconn = sd->dconn;
842 if (dconn) {
843 auto downstream = dconn->get_downstream();
844 auto upstream = downstream->get_upstream();
845
846 if (downstream->get_downstream_stream_id() % 2 == 0 &&
847 downstream->get_request_state() == DownstreamState::INITIAL) {
848 // Downstream is canceled in backend before it is submitted in
849 // frontend session.
850
851 // This will avoid to send RST_STREAM to backend
852 downstream->set_response_state(DownstreamState::MSG_RESET);
853 upstream->cancel_premature_downstream(downstream);
854 } else {
855 if (downstream->get_upgraded() && downstream->get_response_state() ==
856 DownstreamState::HEADER_COMPLETE) {
857 // For tunneled connection, we have to submit RST_STREAM to
858 // upstream *after* whole response body is sent. We just set
859 // MSG_COMPLETE here. Upstream will take care of that.
860 downstream->get_upstream()->on_downstream_body_complete(downstream);
861 downstream->set_response_state(DownstreamState::MSG_COMPLETE);
862 } else if (error_code == NGHTTP2_NO_ERROR) {
863 switch (downstream->get_response_state()) {
864 case DownstreamState::MSG_COMPLETE:
865 case DownstreamState::MSG_BAD_HEADER:
866 break;
867 default:
868 downstream->set_response_state(DownstreamState::MSG_RESET);
869 }
870 } else if (downstream->get_response_state() !=
871 DownstreamState::MSG_BAD_HEADER) {
872 downstream->set_response_state(DownstreamState::MSG_RESET);
873 }
874 if (downstream->get_response_state() == DownstreamState::MSG_RESET &&
875 downstream->get_response_rst_stream_error_code() ==
876 NGHTTP2_NO_ERROR) {
877 downstream->set_response_rst_stream_error_code(error_code);
878 }
879 call_downstream_readcb(http2session, downstream);
880 }
881 // dconn may be deleted
882 }
883 // The life time of StreamData ends here
884 http2session->remove_stream_data(sd);
885 return 0;
886 }
887 } // namespace
888
start_settings_timer()889 void Http2Session::start_settings_timer() {
890 auto &downstreamconf = get_config()->http2.downstream;
891
892 ev_timer_set(&settings_timer_, downstreamconf.timeout.settings, 0.);
893 ev_timer_start(conn_.loop, &settings_timer_);
894 }
895
stop_settings_timer()896 void Http2Session::stop_settings_timer() {
897 ev_timer_stop(conn_.loop, &settings_timer_);
898 }
899
900 namespace {
on_header_callback2(nghttp2_session * session,const nghttp2_frame * frame,nghttp2_rcbuf * name,nghttp2_rcbuf * value,uint8_t flags,void * user_data)901 int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame,
902 nghttp2_rcbuf *name, nghttp2_rcbuf *value,
903 uint8_t flags, void *user_data) {
904 auto http2session = static_cast<Http2Session *>(user_data);
905 auto sd = static_cast<StreamData *>(
906 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
907 if (!sd || !sd->dconn) {
908 return 0;
909 }
910 auto downstream = sd->dconn->get_downstream();
911
912 auto namebuf = nghttp2_rcbuf_get_buf(name);
913 auto valuebuf = nghttp2_rcbuf_get_buf(value);
914
915 auto &resp = downstream->response();
916 auto &httpconf = get_config()->http;
917
918 switch (frame->hd.type) {
919 case NGHTTP2_HEADERS: {
920 auto trailer = frame->headers.cat == NGHTTP2_HCAT_HEADERS &&
921 !downstream->get_expect_final_response();
922
923 if (resp.fs.buffer_size() + namebuf.len + valuebuf.len >
924 httpconf.response_header_field_buffer ||
925 resp.fs.num_fields() >= httpconf.max_response_header_fields) {
926 if (LOG_ENABLED(INFO)) {
927 DLOG(INFO, downstream)
928 << "Too large or many header field size="
929 << resp.fs.buffer_size() + namebuf.len + valuebuf.len
930 << ", num=" << resp.fs.num_fields() + 1;
931 }
932
933 if (trailer) {
934 // We don't care trailer part exceeds header size limit; just
935 // discard it.
936 return 0;
937 }
938
939 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
940 }
941
942 auto token = http2::lookup_token(namebuf.base, namebuf.len);
943 auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX;
944
945 downstream->add_rcbuf(name);
946 downstream->add_rcbuf(value);
947
948 if (trailer) {
949 // just store header fields for trailer part
950 resp.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len},
951 StringRef{valuebuf.base, valuebuf.len},
952 no_index, token);
953 return 0;
954 }
955
956 resp.fs.add_header_token(StringRef{namebuf.base, namebuf.len},
957 StringRef{valuebuf.base, valuebuf.len}, no_index,
958 token);
959 return 0;
960 }
961 case NGHTTP2_PUSH_PROMISE: {
962 auto promised_stream_id = frame->push_promise.promised_stream_id;
963 auto promised_sd = static_cast<StreamData *>(
964 nghttp2_session_get_stream_user_data(session, promised_stream_id));
965 if (!promised_sd || !promised_sd->dconn) {
966 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
967 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
968 }
969
970 auto promised_downstream = promised_sd->dconn->get_downstream();
971
972 auto namebuf = nghttp2_rcbuf_get_buf(name);
973 auto valuebuf = nghttp2_rcbuf_get_buf(value);
974
975 assert(promised_downstream);
976
977 auto &promised_req = promised_downstream->request();
978
979 // We use request header limit for PUSH_PROMISE
980 if (promised_req.fs.buffer_size() + namebuf.len + valuebuf.len >
981 httpconf.request_header_field_buffer ||
982 promised_req.fs.num_fields() >= httpconf.max_request_header_fields) {
983 if (LOG_ENABLED(INFO)) {
984 DLOG(INFO, downstream)
985 << "Too large or many header field size="
986 << promised_req.fs.buffer_size() + namebuf.len + valuebuf.len
987 << ", num=" << promised_req.fs.num_fields() + 1;
988 }
989
990 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
991 }
992
993 promised_downstream->add_rcbuf(name);
994 promised_downstream->add_rcbuf(value);
995
996 auto token = http2::lookup_token(namebuf.base, namebuf.len);
997 promised_req.fs.add_header_token(StringRef{namebuf.base, namebuf.len},
998 StringRef{valuebuf.base, valuebuf.len},
999 flags & NGHTTP2_NV_FLAG_NO_INDEX, token);
1000
1001 return 0;
1002 }
1003 }
1004
1005 return 0;
1006 }
1007 } // namespace
1008
1009 namespace {
on_invalid_header_callback2(nghttp2_session * session,const nghttp2_frame * frame,nghttp2_rcbuf * name,nghttp2_rcbuf * value,uint8_t flags,void * user_data)1010 int on_invalid_header_callback2(nghttp2_session *session,
1011 const nghttp2_frame *frame, nghttp2_rcbuf *name,
1012 nghttp2_rcbuf *value, uint8_t flags,
1013 void *user_data) {
1014 auto http2session = static_cast<Http2Session *>(user_data);
1015 auto sd = static_cast<StreamData *>(
1016 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1017 if (!sd || !sd->dconn) {
1018 return 0;
1019 }
1020
1021 int32_t stream_id;
1022
1023 if (frame->hd.type == NGHTTP2_PUSH_PROMISE) {
1024 stream_id = frame->push_promise.promised_stream_id;
1025 } else {
1026 stream_id = frame->hd.stream_id;
1027 }
1028
1029 if (LOG_ENABLED(INFO)) {
1030 auto namebuf = nghttp2_rcbuf_get_buf(name);
1031 auto valuebuf = nghttp2_rcbuf_get_buf(value);
1032
1033 SSLOG(INFO, http2session)
1034 << "Invalid header field for stream_id=" << stream_id
1035 << " in frame type=" << static_cast<uint32_t>(frame->hd.type)
1036 << ": name=[" << StringRef{namebuf.base, namebuf.len} << "], value=["
1037 << StringRef{valuebuf.base, valuebuf.len} << "]";
1038 }
1039
1040 http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
1041
1042 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1043 }
1044 } // namespace
1045
1046 namespace {
on_begin_headers_callback(nghttp2_session * session,const nghttp2_frame * frame,void * user_data)1047 int on_begin_headers_callback(nghttp2_session *session,
1048 const nghttp2_frame *frame, void *user_data) {
1049 auto http2session = static_cast<Http2Session *>(user_data);
1050
1051 switch (frame->hd.type) {
1052 case NGHTTP2_HEADERS: {
1053 if (frame->headers.cat != NGHTTP2_HCAT_RESPONSE &&
1054 frame->headers.cat != NGHTTP2_HCAT_PUSH_RESPONSE) {
1055 return 0;
1056 }
1057 auto sd = static_cast<StreamData *>(
1058 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1059 if (!sd || !sd->dconn) {
1060 http2session->submit_rst_stream(frame->hd.stream_id,
1061 NGHTTP2_INTERNAL_ERROR);
1062 return 0;
1063 }
1064 return 0;
1065 }
1066 case NGHTTP2_PUSH_PROMISE: {
1067 auto promised_stream_id = frame->push_promise.promised_stream_id;
1068 auto sd = static_cast<StreamData *>(
1069 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1070 if (!sd || !sd->dconn) {
1071 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1072 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1073 }
1074
1075 auto downstream = sd->dconn->get_downstream();
1076
1077 assert(downstream);
1078 assert(downstream->get_downstream_stream_id() == frame->hd.stream_id);
1079
1080 if (http2session->handle_downstream_push_promise(downstream,
1081 promised_stream_id) != 0) {
1082 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1083 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1084 }
1085
1086 return 0;
1087 }
1088 }
1089
1090 return 0;
1091 }
1092 } // namespace
1093
1094 namespace {
on_response_headers(Http2Session * http2session,Downstream * downstream,nghttp2_session * session,const nghttp2_frame * frame)1095 int on_response_headers(Http2Session *http2session, Downstream *downstream,
1096 nghttp2_session *session, const nghttp2_frame *frame) {
1097 int rv;
1098
1099 auto upstream = downstream->get_upstream();
1100 auto handler = upstream->get_client_handler();
1101 const auto &req = downstream->request();
1102 auto &resp = downstream->response();
1103
1104 auto &nva = resp.fs.headers();
1105
1106 auto config = get_config();
1107 auto &loggingconf = config->logging;
1108
1109 downstream->set_expect_final_response(false);
1110
1111 auto status = resp.fs.header(http2::HD__STATUS);
1112 // libnghttp2 guarantees this exists and can be parsed
1113 assert(status);
1114 auto status_code = http2::parse_http_status_code(status->value);
1115
1116 resp.http_status = status_code;
1117 resp.http_major = 2;
1118 resp.http_minor = 0;
1119
1120 downstream->set_downstream_addr_group(
1121 http2session->get_downstream_addr_group());
1122 downstream->set_addr(http2session->get_addr());
1123
1124 if (LOG_ENABLED(INFO)) {
1125 std::stringstream ss;
1126 for (auto &nv : nva) {
1127 ss << TTY_HTTP_HD << nv.name << TTY_RST << ": " << nv.value << "\n";
1128 }
1129 SSLOG(INFO, http2session)
1130 << "HTTP response headers. stream_id=" << frame->hd.stream_id << "\n"
1131 << ss.str();
1132 }
1133
1134 if (downstream->get_non_final_response()) {
1135
1136 if (LOG_ENABLED(INFO)) {
1137 SSLOG(INFO, http2session) << "This is non-final response.";
1138 }
1139
1140 downstream->set_expect_final_response(true);
1141 rv = upstream->on_downstream_header_complete(downstream);
1142
1143 // Now Dowstream's response headers are erased.
1144
1145 if (rv != 0) {
1146 http2session->submit_rst_stream(frame->hd.stream_id,
1147 NGHTTP2_PROTOCOL_ERROR);
1148 downstream->set_response_state(DownstreamState::MSG_RESET);
1149 }
1150
1151 return 0;
1152 }
1153
1154 downstream->set_response_state(DownstreamState::HEADER_COMPLETE);
1155 downstream->check_upgrade_fulfilled_http2();
1156
1157 if (downstream->get_upgraded()) {
1158 resp.connection_close = true;
1159 // On upgrade success, both ends can send data
1160 if (upstream->resume_read(SHRPX_NO_BUFFER, downstream, 0) != 0) {
1161 // If resume_read fails, just drop connection. Not ideal.
1162 delete handler;
1163 return -1;
1164 }
1165 downstream->set_request_state(DownstreamState::HEADER_COMPLETE);
1166 if (LOG_ENABLED(INFO)) {
1167 SSLOG(INFO, http2session)
1168 << "HTTP upgrade success. stream_id=" << frame->hd.stream_id;
1169 }
1170 } else {
1171 auto content_length = resp.fs.header(http2::HD_CONTENT_LENGTH);
1172 if (content_length) {
1173 // libnghttp2 guarantees this can be parsed
1174 resp.fs.content_length = util::parse_uint(content_length->value);
1175 }
1176
1177 if (resp.fs.content_length == -1 && downstream->expect_response_body()) {
1178 // Here we have response body but Content-Length is not known in
1179 // advance.
1180 if (req.http_major <= 0 || (req.http_major == 1 && req.http_minor == 0)) {
1181 // We simply close connection for pre-HTTP/1.1 in this case.
1182 resp.connection_close = true;
1183 } else {
1184 // Otherwise, use chunked encoding to keep upstream connection
1185 // open. In HTTP2, we are supposed not to receive
1186 // transfer-encoding.
1187 resp.fs.add_header_token(StringRef::from_lit("transfer-encoding"),
1188 StringRef::from_lit("chunked"), false,
1189 http2::HD_TRANSFER_ENCODING);
1190 downstream->set_chunked_response(true);
1191 }
1192 }
1193 }
1194
1195 if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1196 resp.headers_only = true;
1197 }
1198
1199 if (loggingconf.access.write_early && downstream->accesslog_ready()) {
1200 handler->write_accesslog(downstream);
1201 downstream->set_accesslog_written(true);
1202 }
1203
1204 rv = upstream->on_downstream_header_complete(downstream);
1205 if (rv != 0) {
1206 // Handling early return (in other words, response was hijacked by
1207 // mruby scripting).
1208 if (downstream->get_response_state() == DownstreamState::MSG_COMPLETE) {
1209 http2session->submit_rst_stream(frame->hd.stream_id, NGHTTP2_CANCEL);
1210 } else {
1211 http2session->submit_rst_stream(frame->hd.stream_id,
1212 NGHTTP2_INTERNAL_ERROR);
1213 downstream->set_response_state(DownstreamState::MSG_RESET);
1214 }
1215 }
1216
1217 return 0;
1218 }
1219 } // namespace
1220
1221 namespace {
on_frame_recv_callback(nghttp2_session * session,const nghttp2_frame * frame,void * user_data)1222 int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame,
1223 void *user_data) {
1224 int rv;
1225 auto http2session = static_cast<Http2Session *>(user_data);
1226
1227 switch (frame->hd.type) {
1228 case NGHTTP2_DATA: {
1229 auto sd = static_cast<StreamData *>(
1230 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1231 if (!sd || !sd->dconn) {
1232 return 0;
1233 }
1234 auto downstream = sd->dconn->get_downstream();
1235 auto upstream = downstream->get_upstream();
1236 rv = upstream->on_downstream_body(downstream, nullptr, 0, true);
1237 if (rv != 0) {
1238 http2session->submit_rst_stream(frame->hd.stream_id,
1239 NGHTTP2_INTERNAL_ERROR);
1240 downstream->set_response_state(DownstreamState::MSG_RESET);
1241
1242 } else if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1243
1244 downstream->disable_downstream_rtimer();
1245
1246 if (downstream->get_response_state() ==
1247 DownstreamState::HEADER_COMPLETE) {
1248
1249 downstream->set_response_state(DownstreamState::MSG_COMPLETE);
1250
1251 rv = upstream->on_downstream_body_complete(downstream);
1252
1253 if (rv != 0) {
1254 downstream->set_response_state(DownstreamState::MSG_RESET);
1255 }
1256 }
1257 }
1258
1259 call_downstream_readcb(http2session, downstream);
1260 return 0;
1261 }
1262 case NGHTTP2_HEADERS: {
1263 auto sd = static_cast<StreamData *>(
1264 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1265 if (!sd || !sd->dconn) {
1266 return 0;
1267 }
1268 auto downstream = sd->dconn->get_downstream();
1269
1270 if (frame->headers.cat == NGHTTP2_HCAT_RESPONSE ||
1271 frame->headers.cat == NGHTTP2_HCAT_PUSH_RESPONSE) {
1272 rv = on_response_headers(http2session, downstream, session, frame);
1273
1274 if (rv != 0) {
1275 return 0;
1276 }
1277 } else if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) {
1278 if (downstream->get_expect_final_response()) {
1279 rv = on_response_headers(http2session, downstream, session, frame);
1280
1281 if (rv != 0) {
1282 return 0;
1283 }
1284 }
1285 }
1286
1287 if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1288 downstream->disable_downstream_rtimer();
1289
1290 if (downstream->get_response_state() ==
1291 DownstreamState::HEADER_COMPLETE) {
1292 downstream->set_response_state(DownstreamState::MSG_COMPLETE);
1293
1294 auto upstream = downstream->get_upstream();
1295
1296 rv = upstream->on_downstream_body_complete(downstream);
1297
1298 if (rv != 0) {
1299 downstream->set_response_state(DownstreamState::MSG_RESET);
1300 }
1301 }
1302 } else {
1303 downstream->reset_downstream_rtimer();
1304 }
1305
1306 // This may delete downstream
1307 call_downstream_readcb(http2session, downstream);
1308
1309 return 0;
1310 }
1311 case NGHTTP2_RST_STREAM: {
1312 auto sd = static_cast<StreamData *>(
1313 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1314 if (sd && sd->dconn) {
1315 auto downstream = sd->dconn->get_downstream();
1316 downstream->set_response_rst_stream_error_code(
1317 frame->rst_stream.error_code);
1318 call_downstream_readcb(http2session, downstream);
1319 }
1320 return 0;
1321 }
1322 case NGHTTP2_SETTINGS: {
1323 if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
1324 http2session->on_settings_received(frame);
1325 return 0;
1326 }
1327
1328 http2session->stop_settings_timer();
1329
1330 auto addr = http2session->get_addr();
1331 auto &connect_blocker = addr->connect_blocker;
1332
1333 connect_blocker->on_success();
1334
1335 return 0;
1336 }
1337 case NGHTTP2_PING:
1338 if (frame->hd.flags & NGHTTP2_FLAG_ACK) {
1339 if (LOG_ENABLED(INFO)) {
1340 LOG(INFO) << "PING ACK received";
1341 }
1342 http2session->connection_alive();
1343 }
1344 return 0;
1345 case NGHTTP2_PUSH_PROMISE: {
1346 auto promised_stream_id = frame->push_promise.promised_stream_id;
1347
1348 if (LOG_ENABLED(INFO)) {
1349 SSLOG(INFO, http2session)
1350 << "Received downstream PUSH_PROMISE stream_id="
1351 << frame->hd.stream_id
1352 << ", promised_stream_id=" << promised_stream_id;
1353 }
1354
1355 auto sd = static_cast<StreamData *>(
1356 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1357 if (!sd || !sd->dconn) {
1358 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1359 return 0;
1360 }
1361
1362 auto downstream = sd->dconn->get_downstream();
1363
1364 assert(downstream);
1365 assert(downstream->get_downstream_stream_id() == frame->hd.stream_id);
1366
1367 auto promised_sd = static_cast<StreamData *>(
1368 nghttp2_session_get_stream_user_data(session, promised_stream_id));
1369 if (!promised_sd || !promised_sd->dconn) {
1370 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1371 return 0;
1372 }
1373
1374 auto promised_downstream = promised_sd->dconn->get_downstream();
1375
1376 assert(promised_downstream);
1377
1378 if (http2session->handle_downstream_push_promise_complete(
1379 downstream, promised_downstream) != 0) {
1380 http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1381 return 0;
1382 }
1383
1384 return 0;
1385 }
1386 case NGHTTP2_GOAWAY:
1387 if (LOG_ENABLED(INFO)) {
1388 auto debug_data = util::ascii_dump(frame->goaway.opaque_data,
1389 frame->goaway.opaque_data_len);
1390
1391 SSLOG(INFO, http2session)
1392 << "GOAWAY received: last-stream-id=" << frame->goaway.last_stream_id
1393 << ", error_code=" << frame->goaway.error_code
1394 << ", debug_data=" << debug_data;
1395 }
1396 return 0;
1397 default:
1398 return 0;
1399 }
1400 }
1401 } // namespace
1402
1403 namespace {
on_data_chunk_recv_callback(nghttp2_session * session,uint8_t flags,int32_t stream_id,const uint8_t * data,size_t len,void * user_data)1404 int on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags,
1405 int32_t stream_id, const uint8_t *data,
1406 size_t len, void *user_data) {
1407 int rv;
1408 auto http2session = static_cast<Http2Session *>(user_data);
1409 auto sd = static_cast<StreamData *>(
1410 nghttp2_session_get_stream_user_data(session, stream_id));
1411 if (!sd || !sd->dconn) {
1412 http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1413
1414 if (http2session->consume(stream_id, len) != 0) {
1415 return NGHTTP2_ERR_CALLBACK_FAILURE;
1416 }
1417
1418 return 0;
1419 }
1420 auto downstream = sd->dconn->get_downstream();
1421 if (!downstream->expect_response_body()) {
1422 http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1423
1424 if (http2session->consume(stream_id, len) != 0) {
1425 return NGHTTP2_ERR_CALLBACK_FAILURE;
1426 }
1427
1428 return 0;
1429 }
1430
1431 // We don't want DATA after non-final response, which is illegal in
1432 // HTTP.
1433 if (downstream->get_non_final_response()) {
1434 http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
1435
1436 if (http2session->consume(stream_id, len) != 0) {
1437 return NGHTTP2_ERR_CALLBACK_FAILURE;
1438 }
1439
1440 return 0;
1441 }
1442
1443 downstream->reset_downstream_rtimer();
1444
1445 auto &resp = downstream->response();
1446
1447 resp.recv_body_length += len;
1448 resp.unconsumed_body_length += len;
1449
1450 auto upstream = downstream->get_upstream();
1451 rv = upstream->on_downstream_body(downstream, data, len, false);
1452 if (rv != 0) {
1453 http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1454
1455 if (http2session->consume(stream_id, len) != 0) {
1456 return NGHTTP2_ERR_CALLBACK_FAILURE;
1457 }
1458
1459 downstream->set_response_state(DownstreamState::MSG_RESET);
1460 }
1461
1462 call_downstream_readcb(http2session, downstream);
1463 return 0;
1464 }
1465 } // namespace
1466
1467 namespace {
on_frame_send_callback(nghttp2_session * session,const nghttp2_frame * frame,void * user_data)1468 int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
1469 void *user_data) {
1470 auto http2session = static_cast<Http2Session *>(user_data);
1471
1472 if (frame->hd.type == NGHTTP2_DATA || frame->hd.type == NGHTTP2_HEADERS) {
1473 auto sd = static_cast<StreamData *>(
1474 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1475
1476 if (!sd || !sd->dconn) {
1477 return 0;
1478 }
1479
1480 auto downstream = sd->dconn->get_downstream();
1481
1482 if (frame->hd.type == NGHTTP2_HEADERS &&
1483 frame->headers.cat == NGHTTP2_HCAT_REQUEST) {
1484 downstream->set_request_header_sent(true);
1485 auto src = downstream->get_blocked_request_buf();
1486 if (src->rleft()) {
1487 auto dest = downstream->get_request_buf();
1488 src->remove(*dest);
1489 if (http2session->resume_data(sd->dconn) != 0) {
1490 return NGHTTP2_ERR_CALLBACK_FAILURE;
1491 }
1492 downstream->ensure_downstream_wtimer();
1493 }
1494 }
1495
1496 if ((frame->hd.flags & NGHTTP2_FLAG_END_STREAM) == 0) {
1497 return 0;
1498 }
1499
1500 downstream->reset_downstream_rtimer();
1501
1502 return 0;
1503 }
1504
1505 if (frame->hd.type == NGHTTP2_SETTINGS &&
1506 (frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
1507 http2session->start_settings_timer();
1508 }
1509 return 0;
1510 }
1511 } // namespace
1512
1513 namespace {
on_frame_not_send_callback(nghttp2_session * session,const nghttp2_frame * frame,int lib_error_code,void * user_data)1514 int on_frame_not_send_callback(nghttp2_session *session,
1515 const nghttp2_frame *frame, int lib_error_code,
1516 void *user_data) {
1517 auto http2session = static_cast<Http2Session *>(user_data);
1518 if (LOG_ENABLED(INFO)) {
1519 SSLOG(INFO, http2session) << "Failed to send control frame type="
1520 << static_cast<uint32_t>(frame->hd.type)
1521 << ", lib_error_code=" << lib_error_code << ": "
1522 << nghttp2_strerror(lib_error_code);
1523 }
1524 if (frame->hd.type != NGHTTP2_HEADERS ||
1525 lib_error_code == NGHTTP2_ERR_STREAM_CLOSED ||
1526 lib_error_code == NGHTTP2_ERR_STREAM_CLOSING) {
1527 return 0;
1528 }
1529
1530 auto sd = static_cast<StreamData *>(
1531 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1532 if (!sd) {
1533 return 0;
1534 }
1535 if (!sd->dconn) {
1536 return 0;
1537 }
1538 auto downstream = sd->dconn->get_downstream();
1539
1540 if (lib_error_code == NGHTTP2_ERR_START_STREAM_NOT_ALLOWED) {
1541 // Migrate to another downstream connection.
1542 auto upstream = downstream->get_upstream();
1543
1544 if (upstream->on_downstream_reset(downstream, false)) {
1545 // This should be done for h1 upstream only. Deleting
1546 // ClientHandler for h2 upstream may lead to crash.
1547 delete upstream->get_client_handler();
1548 }
1549
1550 return 0;
1551 }
1552
1553 // To avoid stream hanging around, flag DownstreamState::MSG_RESET.
1554 downstream->set_response_state(DownstreamState::MSG_RESET);
1555 call_downstream_readcb(http2session, downstream);
1556
1557 return 0;
1558 }
1559 } // namespace
1560
1561 namespace {
1562 constexpr auto PADDING = std::array<uint8_t, 256>{};
1563 } // namespace
1564
1565 namespace {
send_data_callback(nghttp2_session * session,nghttp2_frame * frame,const uint8_t * framehd,size_t length,nghttp2_data_source * source,void * user_data)1566 int send_data_callback(nghttp2_session *session, nghttp2_frame *frame,
1567 const uint8_t *framehd, size_t length,
1568 nghttp2_data_source *source, void *user_data) {
1569 auto http2session = static_cast<Http2Session *>(user_data);
1570 auto sd = static_cast<StreamData *>(
1571 nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1572
1573 if (sd == nullptr) {
1574 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1575 }
1576
1577 auto dconn = sd->dconn;
1578 auto downstream = dconn->get_downstream();
1579 auto input = downstream->get_request_buf();
1580 auto wb = http2session->get_request_buf();
1581
1582 size_t padlen = 0;
1583
1584 wb->append(framehd, 9);
1585 if (frame->data.padlen > 0) {
1586 padlen = frame->data.padlen - 1;
1587 wb->append(static_cast<uint8_t>(padlen));
1588 }
1589
1590 input->remove(*wb, length);
1591
1592 wb->append(PADDING.data(), padlen);
1593
1594 if (input->rleft() == 0) {
1595 downstream->disable_downstream_wtimer();
1596 } else {
1597 downstream->reset_downstream_wtimer();
1598 }
1599
1600 if (length > 0) {
1601 // This is important because it will handle flow control
1602 // stuff.
1603 if (downstream->get_upstream()->resume_read(SHRPX_NO_BUFFER, downstream,
1604 length) != 0) {
1605 // In this case, downstream may be deleted.
1606 return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1607 }
1608
1609 // Here sd->dconn could be nullptr, because
1610 // Upstream::resume_read() may delete downstream which will delete
1611 // dconn. Is this still really true?
1612 }
1613
1614 return 0;
1615 }
1616 } // namespace
1617
create_http2_downstream_callbacks()1618 nghttp2_session_callbacks *create_http2_downstream_callbacks() {
1619 int rv;
1620 nghttp2_session_callbacks *callbacks;
1621
1622 rv = nghttp2_session_callbacks_new(&callbacks);
1623
1624 if (rv != 0) {
1625 return nullptr;
1626 }
1627
1628 nghttp2_session_callbacks_set_on_stream_close_callback(
1629 callbacks, on_stream_close_callback);
1630
1631 nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks,
1632 on_frame_recv_callback);
1633
1634 nghttp2_session_callbacks_set_on_data_chunk_recv_callback(
1635 callbacks, on_data_chunk_recv_callback);
1636
1637 nghttp2_session_callbacks_set_on_frame_send_callback(callbacks,
1638 on_frame_send_callback);
1639
1640 nghttp2_session_callbacks_set_on_frame_not_send_callback(
1641 callbacks, on_frame_not_send_callback);
1642
1643 nghttp2_session_callbacks_set_on_header_callback2(callbacks,
1644 on_header_callback2);
1645
1646 nghttp2_session_callbacks_set_on_invalid_header_callback2(
1647 callbacks, on_invalid_header_callback2);
1648
1649 nghttp2_session_callbacks_set_on_begin_headers_callback(
1650 callbacks, on_begin_headers_callback);
1651
1652 nghttp2_session_callbacks_set_send_data_callback(callbacks,
1653 send_data_callback);
1654
1655 if (get_config()->padding) {
1656 nghttp2_session_callbacks_set_select_padding_callback(
1657 callbacks, http::select_padding_callback);
1658 }
1659
1660 return callbacks;
1661 }
1662
connection_made()1663 int Http2Session::connection_made() {
1664 int rv;
1665
1666 state_ = Http2SessionState::CONNECTED;
1667
1668 on_write_ = &Http2Session::downstream_write;
1669 on_read_ = &Http2Session::downstream_read;
1670
1671 if (addr_->tls) {
1672 const unsigned char *next_proto = nullptr;
1673 unsigned int next_proto_len = 0;
1674
1675 #ifndef OPENSSL_NO_NEXTPROTONEG
1676 SSL_get0_next_proto_negotiated(conn_.tls.ssl, &next_proto, &next_proto_len);
1677 #endif // !OPENSSL_NO_NEXTPROTONEG
1678 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
1679 if (!next_proto) {
1680 SSL_get0_alpn_selected(conn_.tls.ssl, &next_proto, &next_proto_len);
1681 }
1682 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
1683
1684 if (!next_proto) {
1685 downstream_failure(addr_, raddr_);
1686 return -1;
1687 }
1688
1689 auto proto = StringRef{next_proto, next_proto_len};
1690 if (LOG_ENABLED(INFO)) {
1691 SSLOG(INFO, this) << "Negotiated next protocol: " << proto;
1692 }
1693 if (!util::check_h2_is_selected(proto)) {
1694 downstream_failure(addr_, raddr_);
1695 return -1;
1696 }
1697 }
1698
1699 auto config = get_config();
1700 auto &http2conf = config->http2;
1701
1702 rv = nghttp2_session_client_new2(&session_, http2conf.downstream.callbacks,
1703 this, http2conf.downstream.option);
1704
1705 if (rv != 0) {
1706 return -1;
1707 }
1708
1709 std::array<nghttp2_settings_entry, 5> entry;
1710 size_t nentry = 3;
1711 entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
1712 entry[0].value = http2conf.downstream.max_concurrent_streams;
1713
1714 entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
1715 entry[1].value = http2conf.downstream.window_size;
1716
1717 entry[2].settings_id = NGHTTP2_SETTINGS_NO_RFC7540_PRIORITIES;
1718 entry[2].value = 1;
1719
1720 if (http2conf.no_server_push || config->http2_proxy) {
1721 entry[nentry].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
1722 entry[nentry].value = 0;
1723 ++nentry;
1724 }
1725
1726 if (http2conf.downstream.decoder_dynamic_table_size !=
1727 NGHTTP2_DEFAULT_HEADER_TABLE_SIZE) {
1728 entry[nentry].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
1729 entry[nentry].value = http2conf.downstream.decoder_dynamic_table_size;
1730 ++nentry;
1731 }
1732
1733 rv = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, entry.data(),
1734 nentry);
1735 if (rv != 0) {
1736 return -1;
1737 }
1738
1739 rv = nghttp2_session_set_local_window_size(
1740 session_, NGHTTP2_FLAG_NONE, 0,
1741 http2conf.downstream.connection_window_size);
1742 if (rv != 0) {
1743 return -1;
1744 }
1745
1746 reset_connection_check_timer(CONNCHK_TIMEOUT);
1747
1748 submit_pending_requests();
1749
1750 signal_write();
1751 return 0;
1752 }
1753
do_read()1754 int Http2Session::do_read() { return read_(*this); }
do_write()1755 int Http2Session::do_write() { return write_(*this); }
1756
on_read(const uint8_t * data,size_t datalen)1757 int Http2Session::on_read(const uint8_t *data, size_t datalen) {
1758 return on_read_(*this, data, datalen);
1759 }
1760
on_write()1761 int Http2Session::on_write() { return on_write_(*this); }
1762
downstream_read(const uint8_t * data,size_t datalen)1763 int Http2Session::downstream_read(const uint8_t *data, size_t datalen) {
1764 ssize_t rv;
1765
1766 rv = nghttp2_session_mem_recv(session_, data, datalen);
1767 if (rv < 0) {
1768 SSLOG(ERROR, this) << "nghttp2_session_mem_recv() returned error: "
1769 << nghttp2_strerror(rv);
1770 return -1;
1771 }
1772
1773 if (nghttp2_session_want_read(session_) == 0 &&
1774 nghttp2_session_want_write(session_) == 0 && wb_.rleft() == 0) {
1775 if (LOG_ENABLED(INFO)) {
1776 SSLOG(INFO, this) << "No more read/write for this HTTP2 session";
1777 }
1778 return -1;
1779 }
1780
1781 signal_write();
1782 return 0;
1783 }
1784
downstream_write()1785 int Http2Session::downstream_write() {
1786 for (;;) {
1787 const uint8_t *data;
1788 auto datalen = nghttp2_session_mem_send(session_, &data);
1789 if (datalen < 0) {
1790 SSLOG(ERROR, this) << "nghttp2_session_mem_send() returned error: "
1791 << nghttp2_strerror(datalen);
1792 return -1;
1793 }
1794 if (datalen == 0) {
1795 break;
1796 }
1797 wb_.append(data, datalen);
1798
1799 if (wb_.rleft() >= MAX_BUFFER_SIZE) {
1800 break;
1801 }
1802 }
1803
1804 if (nghttp2_session_want_read(session_) == 0 &&
1805 nghttp2_session_want_write(session_) == 0 && wb_.rleft() == 0) {
1806 if (LOG_ENABLED(INFO)) {
1807 SSLOG(INFO, this) << "No more read/write for this session";
1808 }
1809 return -1;
1810 }
1811
1812 return 0;
1813 }
1814
signal_write()1815 void Http2Session::signal_write() {
1816 switch (state_) {
1817 case Http2SessionState::DISCONNECTED:
1818 if (!ev_is_active(&initiate_connection_timer_)) {
1819 if (LOG_ENABLED(INFO)) {
1820 LOG(INFO) << "Start connecting to backend server";
1821 }
1822 // Since the timer is set to 0., these will feed 2 events. We
1823 // will stop the timer in the initiate_connection_timer_ to void
1824 // 2nd event.
1825 ev_timer_start(conn_.loop, &initiate_connection_timer_);
1826 ev_feed_event(conn_.loop, &initiate_connection_timer_, 0);
1827 }
1828 break;
1829 case Http2SessionState::CONNECTED:
1830 conn_.wlimit.startw();
1831 break;
1832 default:
1833 break;
1834 }
1835 }
1836
get_loop() const1837 struct ev_loop *Http2Session::get_loop() const {
1838 return conn_.loop;
1839 }
1840
get_wev()1841 ev_io *Http2Session::get_wev() { return &conn_.wev; }
1842
get_state() const1843 Http2SessionState Http2Session::get_state() const { return state_; }
1844
set_state(Http2SessionState state)1845 void Http2Session::set_state(Http2SessionState state) { state_ = state; }
1846
terminate_session(uint32_t error_code)1847 int Http2Session::terminate_session(uint32_t error_code) {
1848 int rv;
1849 rv = nghttp2_session_terminate_session(session_, error_code);
1850 if (rv != 0) {
1851 return -1;
1852 }
1853 return 0;
1854 }
1855
get_ssl() const1856 SSL *Http2Session::get_ssl() const { return conn_.tls.ssl; }
1857
consume(int32_t stream_id,size_t len)1858 int Http2Session::consume(int32_t stream_id, size_t len) {
1859 int rv;
1860
1861 if (!session_) {
1862 return 0;
1863 }
1864
1865 rv = nghttp2_session_consume(session_, stream_id, len);
1866
1867 if (rv != 0) {
1868 SSLOG(WARN, this) << "nghttp2_session_consume() returned error: "
1869 << nghttp2_strerror(rv);
1870
1871 return -1;
1872 }
1873
1874 return 0;
1875 }
1876
can_push_request(const Downstream * downstream) const1877 bool Http2Session::can_push_request(const Downstream *downstream) const {
1878 auto &req = downstream->request();
1879 return state_ == Http2SessionState::CONNECTED &&
1880 connection_check_state_ == ConnectionCheck::NONE &&
1881 (req.connect_proto == ConnectProto::NONE || settings_recved_);
1882 }
1883
start_checking_connection()1884 void Http2Session::start_checking_connection() {
1885 if (state_ != Http2SessionState::CONNECTED ||
1886 connection_check_state_ != ConnectionCheck::REQUIRED) {
1887 return;
1888 }
1889 connection_check_state_ = ConnectionCheck::STARTED;
1890
1891 SSLOG(INFO, this) << "Start checking connection";
1892 // If connection is down, we may get error when writing data. Issue
1893 // ping frame to see whether connection is alive.
1894 nghttp2_submit_ping(session_, NGHTTP2_FLAG_NONE, nullptr);
1895
1896 // set ping timeout and start timer again
1897 reset_connection_check_timer(CONNCHK_PING_TIMEOUT);
1898
1899 signal_write();
1900 }
1901
reset_connection_check_timer(ev_tstamp t)1902 void Http2Session::reset_connection_check_timer(ev_tstamp t) {
1903 connchk_timer_.repeat = t;
1904 ev_timer_again(conn_.loop, &connchk_timer_);
1905 }
1906
reset_connection_check_timer_if_not_checking()1907 void Http2Session::reset_connection_check_timer_if_not_checking() {
1908 if (connection_check_state_ != ConnectionCheck::NONE) {
1909 return;
1910 }
1911
1912 reset_connection_check_timer(CONNCHK_TIMEOUT);
1913 }
1914
connection_alive()1915 void Http2Session::connection_alive() {
1916 reset_connection_check_timer(CONNCHK_TIMEOUT);
1917
1918 if (connection_check_state_ == ConnectionCheck::NONE) {
1919 return;
1920 }
1921
1922 if (LOG_ENABLED(INFO)) {
1923 SSLOG(INFO, this) << "Connection alive";
1924 }
1925
1926 connection_check_state_ = ConnectionCheck::NONE;
1927
1928 submit_pending_requests();
1929 }
1930
submit_pending_requests()1931 void Http2Session::submit_pending_requests() {
1932 for (auto dconn = dconns_.head; dconn; dconn = dconn->dlnext) {
1933 auto downstream = dconn->get_downstream();
1934
1935 if (!downstream->get_request_pending() ||
1936 !downstream->request_submission_ready()) {
1937 continue;
1938 }
1939
1940 auto &req = downstream->request();
1941 if (req.connect_proto != ConnectProto::NONE && !settings_recved_) {
1942 continue;
1943 }
1944
1945 auto upstream = downstream->get_upstream();
1946
1947 if (dconn->push_request_headers() != 0) {
1948 if (LOG_ENABLED(INFO)) {
1949 SSLOG(INFO, this) << "backend request failed";
1950 }
1951
1952 upstream->on_downstream_abort_request(downstream, 400);
1953
1954 continue;
1955 }
1956
1957 upstream->resume_read(SHRPX_NO_BUFFER, downstream, 0);
1958 }
1959 }
1960
set_connection_check_state(ConnectionCheck state)1961 void Http2Session::set_connection_check_state(ConnectionCheck state) {
1962 connection_check_state_ = state;
1963 }
1964
get_connection_check_state() const1965 ConnectionCheck Http2Session::get_connection_check_state() const {
1966 return connection_check_state_;
1967 }
1968
noop()1969 int Http2Session::noop() { return 0; }
1970
read_noop(const uint8_t * data,size_t datalen)1971 int Http2Session::read_noop(const uint8_t *data, size_t datalen) { return 0; }
1972
write_noop()1973 int Http2Session::write_noop() { return 0; }
1974
connected()1975 int Http2Session::connected() {
1976 auto sock_error = util::get_socket_error(conn_.fd);
1977 if (sock_error != 0) {
1978 SSLOG(WARN, this) << "Backend connect failed; addr="
1979 << util::to_numeric_addr(raddr_)
1980 << ": errno=" << sock_error;
1981
1982 downstream_failure(addr_, raddr_);
1983
1984 return -1;
1985 }
1986
1987 if (LOG_ENABLED(INFO)) {
1988 SSLOG(INFO, this) << "Connection established";
1989 }
1990
1991 // Reset timeout for write. Previously, we set timeout for connect.
1992 conn_.wt.repeat = group_->shared_addr->timeout.write;
1993 ev_timer_again(conn_.loop, &conn_.wt);
1994
1995 conn_.rlimit.startw();
1996 conn_.again_rt();
1997
1998 read_ = &Http2Session::read_clear;
1999 write_ = &Http2Session::write_clear;
2000
2001 if (state_ == Http2SessionState::PROXY_CONNECTING) {
2002 return do_write();
2003 }
2004
2005 if (conn_.tls.ssl) {
2006 read_ = &Http2Session::tls_handshake;
2007 write_ = &Http2Session::tls_handshake;
2008
2009 return do_write();
2010 }
2011
2012 if (connection_made() != 0) {
2013 state_ = Http2SessionState::CONNECT_FAILING;
2014 return -1;
2015 }
2016
2017 return 0;
2018 }
2019
read_clear()2020 int Http2Session::read_clear() {
2021 conn_.last_read = ev_now(conn_.loop);
2022
2023 std::array<uint8_t, 16_k> buf;
2024
2025 for (;;) {
2026 auto nread = conn_.read_clear(buf.data(), buf.size());
2027
2028 if (nread == 0) {
2029 return write_clear();
2030 }
2031
2032 if (nread < 0) {
2033 return nread;
2034 }
2035
2036 if (on_read(buf.data(), nread) != 0) {
2037 return -1;
2038 }
2039 }
2040 }
2041
write_clear()2042 int Http2Session::write_clear() {
2043 conn_.last_read = ev_now(conn_.loop);
2044
2045 std::array<struct iovec, MAX_WR_IOVCNT> iov;
2046
2047 for (;;) {
2048 if (wb_.rleft() > 0) {
2049 auto iovcnt = wb_.riovec(iov.data(), iov.size());
2050 auto nwrite = conn_.writev_clear(iov.data(), iovcnt);
2051
2052 if (nwrite == 0) {
2053 return 0;
2054 }
2055
2056 if (nwrite < 0) {
2057 // We may have pending data in receive buffer which may
2058 // contain part of response body. So keep reading. Invoke
2059 // read event to get read(2) error just in case.
2060 ev_feed_event(conn_.loop, &conn_.rev, EV_READ);
2061 write_ = &Http2Session::write_void;
2062 break;
2063 }
2064
2065 wb_.drain(nwrite);
2066 continue;
2067 }
2068
2069 if (on_write() != 0) {
2070 return -1;
2071 }
2072 if (wb_.rleft() == 0) {
2073 break;
2074 }
2075 }
2076
2077 conn_.wlimit.stopw();
2078 ev_timer_stop(conn_.loop, &conn_.wt);
2079
2080 return 0;
2081 }
2082
tls_handshake()2083 int Http2Session::tls_handshake() {
2084 conn_.last_read = ev_now(conn_.loop);
2085
2086 ERR_clear_error();
2087
2088 auto rv = conn_.tls_handshake();
2089
2090 if (rv == SHRPX_ERR_INPROGRESS) {
2091 return 0;
2092 }
2093
2094 if (rv < 0) {
2095 downstream_failure(addr_, raddr_);
2096
2097 return rv;
2098 }
2099
2100 if (LOG_ENABLED(INFO)) {
2101 SSLOG(INFO, this) << "SSL/TLS handshake completed";
2102 }
2103
2104 if (!get_config()->tls.insecure &&
2105 tls::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) {
2106 downstream_failure(addr_, raddr_);
2107
2108 return -1;
2109 }
2110
2111 read_ = &Http2Session::read_tls;
2112 write_ = &Http2Session::write_tls;
2113
2114 if (connection_made() != 0) {
2115 state_ = Http2SessionState::CONNECT_FAILING;
2116 return -1;
2117 }
2118
2119 return 0;
2120 }
2121
read_tls()2122 int Http2Session::read_tls() {
2123 conn_.last_read = ev_now(conn_.loop);
2124
2125 std::array<uint8_t, 16_k> buf;
2126
2127 ERR_clear_error();
2128
2129 for (;;) {
2130 auto nread = conn_.read_tls(buf.data(), buf.size());
2131
2132 if (nread == 0) {
2133 return write_tls();
2134 }
2135
2136 if (nread < 0) {
2137 return nread;
2138 }
2139
2140 if (on_read(buf.data(), nread) != 0) {
2141 return -1;
2142 }
2143 }
2144 }
2145
write_tls()2146 int Http2Session::write_tls() {
2147 conn_.last_read = ev_now(conn_.loop);
2148
2149 ERR_clear_error();
2150
2151 struct iovec iov;
2152
2153 for (;;) {
2154 if (wb_.rleft() > 0) {
2155 auto iovcnt = wb_.riovec(&iov, 1);
2156 if (iovcnt != 1) {
2157 assert(0);
2158 return -1;
2159 }
2160 auto nwrite = conn_.write_tls(iov.iov_base, iov.iov_len);
2161
2162 if (nwrite == 0) {
2163 return 0;
2164 }
2165
2166 if (nwrite < 0) {
2167 // We may have pending data in receive buffer which may
2168 // contain part of response body. So keep reading. Invoke
2169 // read event to get read(2) error just in case.
2170 ev_feed_event(conn_.loop, &conn_.rev, EV_READ);
2171 write_ = &Http2Session::write_void;
2172 break;
2173 }
2174
2175 wb_.drain(nwrite);
2176
2177 continue;
2178 }
2179
2180 if (on_write() != 0) {
2181 return -1;
2182 }
2183 if (wb_.rleft() == 0) {
2184 conn_.start_tls_write_idle();
2185 break;
2186 }
2187 }
2188
2189 conn_.wlimit.stopw();
2190 ev_timer_stop(conn_.loop, &conn_.wt);
2191
2192 return 0;
2193 }
2194
write_void()2195 int Http2Session::write_void() {
2196 conn_.wlimit.stopw();
2197 return 0;
2198 }
2199
should_hard_fail() const2200 bool Http2Session::should_hard_fail() const {
2201 switch (state_) {
2202 case Http2SessionState::PROXY_CONNECTING:
2203 case Http2SessionState::PROXY_FAILED:
2204 return true;
2205 case Http2SessionState::DISCONNECTED: {
2206 const auto &proxy = get_config()->downstream_http_proxy;
2207 return !proxy.host.empty();
2208 }
2209 default:
2210 return false;
2211 }
2212 }
2213
get_addr() const2214 DownstreamAddr *Http2Session::get_addr() const { return addr_; }
2215
handle_downstream_push_promise(Downstream * downstream,int32_t promised_stream_id)2216 int Http2Session::handle_downstream_push_promise(Downstream *downstream,
2217 int32_t promised_stream_id) {
2218 auto upstream = downstream->get_upstream();
2219 if (!upstream->push_enabled()) {
2220 return -1;
2221 }
2222
2223 auto promised_downstream =
2224 upstream->on_downstream_push_promise(downstream, promised_stream_id);
2225 if (!promised_downstream) {
2226 return -1;
2227 }
2228
2229 // Now we have Downstream object for pushed stream.
2230 // promised_downstream->get_stream() still returns 0.
2231
2232 auto handler = upstream->get_client_handler();
2233
2234 auto promised_dconn = std::make_unique<Http2DownstreamConnection>(this);
2235 promised_dconn->set_client_handler(handler);
2236
2237 auto ptr = promised_dconn.get();
2238
2239 if (promised_downstream->attach_downstream_connection(
2240 std::move(promised_dconn)) != 0) {
2241 return -1;
2242 }
2243
2244 auto promised_sd = std::make_unique<StreamData>();
2245
2246 nghttp2_session_set_stream_user_data(session_, promised_stream_id,
2247 promised_sd.get());
2248
2249 ptr->attach_stream_data(promised_sd.get());
2250 streams_.append(promised_sd.release());
2251
2252 return 0;
2253 }
2254
handle_downstream_push_promise_complete(Downstream * downstream,Downstream * promised_downstream)2255 int Http2Session::handle_downstream_push_promise_complete(
2256 Downstream *downstream, Downstream *promised_downstream) {
2257 auto &promised_req = promised_downstream->request();
2258
2259 auto &promised_balloc = promised_downstream->get_block_allocator();
2260
2261 auto authority = promised_req.fs.header(http2::HD__AUTHORITY);
2262 auto path = promised_req.fs.header(http2::HD__PATH);
2263 auto method = promised_req.fs.header(http2::HD__METHOD);
2264 auto scheme = promised_req.fs.header(http2::HD__SCHEME);
2265
2266 if (!authority) {
2267 authority = promised_req.fs.header(http2::HD_HOST);
2268 }
2269
2270 auto method_token = http2::lookup_method_token(method->value);
2271 if (method_token == -1) {
2272 if (LOG_ENABLED(INFO)) {
2273 SSLOG(INFO, this) << "Unrecognized method: " << method->value;
2274 }
2275
2276 return -1;
2277 }
2278
2279 // TODO Rewrite authority if we enabled rewrite host. But we
2280 // really don't know how to rewrite host. Should we use the same
2281 // host in associated stream?
2282 if (authority) {
2283 promised_req.authority = authority->value;
2284 }
2285 promised_req.method = method_token;
2286 // libnghttp2 ensures that we don't have CONNECT method in
2287 // PUSH_PROMISE, and guarantees that :scheme exists.
2288 if (scheme) {
2289 promised_req.scheme = scheme->value;
2290 }
2291
2292 // For server-wide OPTIONS request, path is empty.
2293 if (method_token != HTTP_OPTIONS || path->value != "*") {
2294 promised_req.path = http2::rewrite_clean_path(promised_balloc, path->value);
2295 }
2296
2297 promised_downstream->inspect_http2_request();
2298
2299 auto upstream = promised_downstream->get_upstream();
2300
2301 promised_downstream->set_request_state(DownstreamState::MSG_COMPLETE);
2302 promised_downstream->set_request_header_sent(true);
2303
2304 if (upstream->on_downstream_push_promise_complete(downstream,
2305 promised_downstream) != 0) {
2306 return -1;
2307 }
2308
2309 return 0;
2310 }
2311
get_num_dconns() const2312 size_t Http2Session::get_num_dconns() const { return dconns_.size(); }
2313
max_concurrency_reached(size_t extra) const2314 bool Http2Session::max_concurrency_reached(size_t extra) const {
2315 if (!session_) {
2316 return dconns_.size() + extra >= 100;
2317 }
2318
2319 // If session does not allow further requests, it effectively means
2320 // that maximum concurrency is reached.
2321 return !nghttp2_session_check_request_allowed(session_) ||
2322 dconns_.size() + extra >=
2323 nghttp2_session_get_remote_settings(
2324 session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS);
2325 }
2326
2327 const std::shared_ptr<DownstreamAddrGroup> &
get_downstream_addr_group() const2328 Http2Session::get_downstream_addr_group() const {
2329 return group_;
2330 }
2331
add_to_extra_freelist()2332 void Http2Session::add_to_extra_freelist() {
2333 if (freelist_zone_ != FreelistZone::NONE) {
2334 return;
2335 }
2336
2337 if (LOG_ENABLED(INFO)) {
2338 SSLOG(INFO, this) << "Append to http2_extra_freelist, addr=" << addr_
2339 << ", freelist.size="
2340 << addr_->http2_extra_freelist.size();
2341 }
2342
2343 freelist_zone_ = FreelistZone::EXTRA;
2344 addr_->http2_extra_freelist.append(this);
2345 }
2346
remove_from_freelist()2347 void Http2Session::remove_from_freelist() {
2348 switch (freelist_zone_) {
2349 case FreelistZone::NONE:
2350 return;
2351 case FreelistZone::EXTRA:
2352 if (LOG_ENABLED(INFO)) {
2353 SSLOG(INFO, this) << "Remove from http2_extra_freelist, addr=" << addr_
2354 << ", freelist.size="
2355 << addr_->http2_extra_freelist.size();
2356 }
2357 addr_->http2_extra_freelist.remove(this);
2358 break;
2359 case FreelistZone::GONE:
2360 return;
2361 }
2362
2363 freelist_zone_ = FreelistZone::NONE;
2364 }
2365
exclude_from_scheduling()2366 void Http2Session::exclude_from_scheduling() {
2367 remove_from_freelist();
2368 freelist_zone_ = FreelistZone::GONE;
2369 }
2370
get_request_buf()2371 DefaultMemchunks *Http2Session::get_request_buf() { return &wb_; }
2372
on_timeout()2373 void Http2Session::on_timeout() {
2374 switch (state_) {
2375 case Http2SessionState::PROXY_CONNECTING: {
2376 auto worker_blocker = worker_->get_connect_blocker();
2377 worker_blocker->on_failure();
2378 break;
2379 }
2380 case Http2SessionState::CONNECTING:
2381 SSLOG(WARN, this) << "Connect time out; addr="
2382 << util::to_numeric_addr(raddr_);
2383
2384 downstream_failure(addr_, raddr_);
2385 break;
2386 default:
2387 break;
2388 }
2389 }
2390
check_retire()2391 void Http2Session::check_retire() {
2392 if (!group_->retired) {
2393 return;
2394 }
2395
2396 ev_prepare_stop(conn_.loop, &prep_);
2397
2398 if (!session_) {
2399 return;
2400 }
2401
2402 auto last_stream_id = nghttp2_session_get_last_proc_stream_id(session_);
2403 nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, last_stream_id,
2404 NGHTTP2_NO_ERROR, nullptr, 0);
2405
2406 signal_write();
2407 }
2408
get_raddr() const2409 const Address *Http2Session::get_raddr() const { return raddr_; }
2410
on_settings_received(const nghttp2_frame * frame)2411 void Http2Session::on_settings_received(const nghttp2_frame *frame) {
2412 // TODO This effectively disallows nghttpx to change its behaviour
2413 // based on the 2nd SETTINGS.
2414 if (settings_recved_) {
2415 return;
2416 }
2417
2418 settings_recved_ = true;
2419
2420 for (size_t i = 0; i < frame->settings.niv; ++i) {
2421 auto &ent = frame->settings.iv[i];
2422 if (ent.settings_id == NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL) {
2423 allow_connect_proto_ = true;
2424 break;
2425 }
2426 }
2427
2428 submit_pending_requests();
2429 }
2430
get_allow_connect_proto() const2431 bool Http2Session::get_allow_connect_proto() const {
2432 return allow_connect_proto_;
2433 }
2434
2435 } // namespace shrpx
2436