• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * nghttp2 - HTTP/2 C Library
3  *
4  * Copyright (c) 2021 Tatsuhiro Tsujikawa
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining
7  * a copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sublicense, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice shall be
15  * included in all copies or substantial portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  */
25 #include "shrpx_quic_connection_handler.h"
26 
27 #include <openssl/rand.h>
28 
29 #include <ngtcp2/ngtcp2.h>
30 #include <ngtcp2/ngtcp2_crypto.h>
31 
32 #include "shrpx_worker.h"
33 #include "shrpx_client_handler.h"
34 #include "shrpx_log.h"
35 #include "shrpx_http3_upstream.h"
36 #include "shrpx_connection_handler.h"
37 
38 namespace shrpx {
39 
40 namespace {
stateless_reset_bucket_regen_timercb(struct ev_loop * loop,ev_timer * w,int revents)41 void stateless_reset_bucket_regen_timercb(struct ev_loop *loop, ev_timer *w,
42                                           int revents) {
43   auto quic_conn_handler = static_cast<QUICConnectionHandler *>(w->data);
44 
45   quic_conn_handler->on_stateless_reset_bucket_regen();
46 }
47 } // namespace
48 
QUICConnectionHandler(Worker * worker)49 QUICConnectionHandler::QUICConnectionHandler(Worker *worker)
50     : worker_{worker},
51       stateless_reset_bucket_{SHRPX_QUIC_STATELESS_RESET_BURST} {
52   ev_timer_init(&stateless_reset_bucket_regen_timer_,
53                 stateless_reset_bucket_regen_timercb, 0., 1.);
54   stateless_reset_bucket_regen_timer_.data = this;
55 }
56 
~QUICConnectionHandler()57 QUICConnectionHandler::~QUICConnectionHandler() {
58   ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
59 }
60 
handle_packet(const UpstreamAddr * faddr,const Address & remote_addr,const Address & local_addr,const ngtcp2_pkt_info & pi,const uint8_t * data,size_t datalen)61 int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
62                                          const Address &remote_addr,
63                                          const Address &local_addr,
64                                          const ngtcp2_pkt_info &pi,
65                                          const uint8_t *data, size_t datalen) {
66   int rv;
67   ngtcp2_version_cid vc;
68 
69   rv = ngtcp2_pkt_decode_version_cid(&vc, data, datalen, SHRPX_QUIC_SCIDLEN);
70   switch (rv) {
71   case 0:
72     break;
73   case NGTCP2_ERR_VERSION_NEGOTIATION:
74     send_version_negotiation(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
75                              vc.scidlen, remote_addr, local_addr);
76 
77     return 0;
78   default:
79     return 0;
80   }
81 
82   auto config = get_config();
83 
84   ngtcp2_cid dcid_key;
85   ngtcp2_cid_init(&dcid_key, vc.dcid, vc.dcidlen);
86 
87   auto conn_handler = worker_->get_connection_handler();
88 
89   ClientHandler *handler;
90 
91   auto &quicconf = config->quic;
92 
93   auto it = connections_.find(dcid_key);
94   if (it == std::end(connections_)) {
95     auto cwit = close_waits_.find(dcid_key);
96     if (cwit != std::end(close_waits_)) {
97       auto cw = (*cwit).second;
98 
99       cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
100 
101       return 0;
102     }
103 
104     if (data[0] & 0x80) {
105       if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr,
106                                              dcid_key) != 0) {
107         return 0;
108       }
109 
110       it = connections_.find(dcid_key);
111       if (it == std::end(connections_)) {
112         auto cwit = close_waits_.find(dcid_key);
113         if (cwit != std::end(close_waits_)) {
114           auto cw = (*cwit).second;
115 
116           cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
117 
118           return 0;
119         }
120       }
121     }
122   }
123 
124   if (it == std::end(connections_)) {
125     std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
126 
127     auto &qkms = conn_handler->get_quic_keying_materials();
128     const QUICKeyingMaterial *qkm = nullptr;
129 
130     if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) {
131       qkm = select_quic_keying_material(
132           *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK);
133 
134       if (decrypt_quic_connection_id(decrypted_dcid.data(),
135                                      vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
136                                      qkm->cid_encryption_key.data()) != 0) {
137         return 0;
138       }
139 
140       if (qkm != &qkms->keying_materials.front() ||
141           !std::equal(std::begin(decrypted_dcid),
142                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
143                       worker_->get_cid_prefix())) {
144         auto quic_lwp =
145             conn_handler->match_quic_lingering_worker_process_cid_prefix(
146                 decrypted_dcid.data(), decrypted_dcid.size());
147         if (quic_lwp) {
148           if (conn_handler->forward_quic_packet_to_lingering_worker_process(
149                   quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) {
150             return 0;
151           }
152 
153           return 0;
154         }
155       }
156     }
157 
158     // new connection
159 
160     auto &upstreamconf = config->conn.upstream;
161     if (worker_->get_worker_stat()->num_connections >=
162         upstreamconf.worker_connections) {
163       if (LOG_ENABLED(INFO)) {
164         LOG(INFO) << "Too many connections >="
165                   << upstreamconf.worker_connections;
166       }
167 
168       return 0;
169     }
170 
171     ngtcp2_pkt_hd hd;
172     ngtcp2_cid odcid, *podcid = nullptr;
173     const uint8_t *token = nullptr;
174     size_t tokenlen = 0;
175 
176     switch (ngtcp2_accept(&hd, data, datalen)) {
177     case 0: {
178       // If we get Initial and it has the CID prefix of this worker,
179       // it is likely that client is intentionally use the prefix.
180       // Just drop it.
181       if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) {
182         if (qkm != &qkms->keying_materials.front()) {
183           qkm = &qkms->keying_materials.front();
184 
185           if (decrypt_quic_connection_id(decrypted_dcid.data(),
186                                          vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
187                                          qkm->cid_encryption_key.data()) != 0) {
188             return 0;
189           }
190         }
191 
192         if (std::equal(std::begin(decrypted_dcid),
193                        std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
194                        worker_->get_cid_prefix())) {
195           return 0;
196         }
197       }
198 
199       if (worker_->get_graceful_shutdown()) {
200         send_connection_close(faddr, hd.version, hd.dcid, hd.scid, remote_addr,
201                               local_addr, NGTCP2_CONNECTION_REFUSED,
202                               datalen * 3);
203         return 0;
204       }
205 
206       if (hd.tokenlen == 0) {
207         if (quicconf.upstream.require_token) {
208           send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
209                      vc.scidlen, remote_addr, local_addr, datalen * 3);
210 
211           return 0;
212         }
213 
214         break;
215       }
216 
217       switch (hd.token[0]) {
218       case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY: {
219         if (vc.dcidlen != SHRPX_QUIC_SCIDLEN) {
220           // Initial packets with Retry token must have DCID chosen by
221           // server.
222           return 0;
223         }
224 
225         auto qkm = select_quic_keying_material(
226             *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK);
227 
228         if (verify_retry_token(odcid, hd.token, hd.tokenlen, hd.version,
229                                hd.dcid, &remote_addr.su.sa, remote_addr.len,
230                                qkm->secret.data(), qkm->secret.size()) != 0) {
231           if (LOG_ENABLED(INFO)) {
232             LOG(INFO) << "Failed to validate Retry token from remote="
233                       << util::to_numeric_addr(&remote_addr);
234           }
235 
236           // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE
237           // with INVALID_TOKEN.
238           send_connection_close(faddr, hd.version, hd.dcid, hd.scid,
239                                 remote_addr, local_addr, NGTCP2_INVALID_TOKEN,
240                                 datalen * 3);
241           return 0;
242         }
243 
244         if (LOG_ENABLED(INFO)) {
245           LOG(INFO) << "Successfully validated Retry token from remote="
246                     << util::to_numeric_addr(&remote_addr);
247         }
248 
249         podcid = &odcid;
250         token = hd.token;
251         tokenlen = hd.tokenlen;
252 
253         break;
254       }
255       case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR: {
256         // If a token is a regular token, it must be at least
257         // NGTCP2_MIN_INITIAL_DCIDLEN bytes long.
258         if (vc.dcidlen < NGTCP2_MIN_INITIAL_DCIDLEN) {
259           return 0;
260         }
261 
262         if (hd.tokenlen != NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN + 1) {
263           if (LOG_ENABLED(INFO)) {
264             LOG(INFO) << "Failed to validate token from remote="
265                       << util::to_numeric_addr(&remote_addr);
266           }
267 
268           if (quicconf.upstream.require_token) {
269             send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
270                        vc.scidlen, remote_addr, local_addr, datalen * 3);
271 
272             return 0;
273           }
274 
275           break;
276         }
277 
278         auto qkm = select_quic_keying_material(
279             *qkms.get(), hd.token[NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN]);
280 
281         if (verify_token(hd.token, hd.tokenlen - 1, &remote_addr.su.sa,
282                          remote_addr.len, qkm->secret.data(),
283                          qkm->secret.size()) != 0) {
284           if (LOG_ENABLED(INFO)) {
285             LOG(INFO) << "Failed to validate token from remote="
286                       << util::to_numeric_addr(&remote_addr);
287           }
288 
289           if (quicconf.upstream.require_token) {
290             send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
291                        vc.scidlen, remote_addr, local_addr, datalen * 3);
292 
293             return 0;
294           }
295 
296           break;
297         }
298 
299         if (LOG_ENABLED(INFO)) {
300           LOG(INFO) << "Successfully validated token from remote="
301                     << util::to_numeric_addr(&remote_addr);
302         }
303 
304         token = hd.token;
305         tokenlen = hd.tokenlen;
306 
307         break;
308       }
309       default:
310         if (quicconf.upstream.require_token) {
311           send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
312                      vc.scidlen, remote_addr, local_addr, datalen * 3);
313 
314           return 0;
315         }
316 
317         break;
318       }
319 
320       break;
321     }
322     default:
323       if (!config->single_thread && !(data[0] & 0x80) &&
324           vc.dcidlen == SHRPX_QUIC_SCIDLEN &&
325           !std::equal(std::begin(decrypted_dcid),
326                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
327                       worker_->get_cid_prefix())) {
328         if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr,
329                                               pi, decrypted_dcid.data(), data,
330                                               datalen) == 0) {
331           return 0;
332         }
333       }
334 
335       if (!(data[0] & 0x80)) {
336         // TODO Must be rate limited
337         send_stateless_reset(faddr, vc.dcid, vc.dcidlen, remote_addr,
338                              local_addr);
339       }
340 
341       return 0;
342     }
343 
344     handler = handle_new_connection(faddr, remote_addr, local_addr, hd, podcid,
345                                     token, tokenlen);
346     if (handler == nullptr) {
347       return 0;
348     }
349   } else {
350     handler = (*it).second;
351   }
352 
353   if (handler->read_quic(faddr, remote_addr, local_addr, pi, data, datalen) !=
354       0) {
355     delete handler;
356     return 0;
357   }
358 
359   handler->signal_write();
360 
361   return 0;
362 }
363 
handle_new_connection(const UpstreamAddr * faddr,const Address & remote_addr,const Address & local_addr,const ngtcp2_pkt_hd & hd,const ngtcp2_cid * odcid,const uint8_t * token,size_t tokenlen)364 ClientHandler *QUICConnectionHandler::handle_new_connection(
365     const UpstreamAddr *faddr, const Address &remote_addr,
366     const Address &local_addr, const ngtcp2_pkt_hd &hd, const ngtcp2_cid *odcid,
367     const uint8_t *token, size_t tokenlen) {
368   std::array<char, NI_MAXHOST> host;
369   std::array<char, NI_MAXSERV> service;
370   int rv;
371 
372   rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
373                    host.size(), service.data(), service.size(),
374                    NI_NUMERICHOST | NI_NUMERICSERV);
375   if (rv != 0) {
376     LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
377 
378     return nullptr;
379   }
380 
381   auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
382 
383   assert(ssl_ctx);
384 
385   auto ssl = tls::create_ssl(ssl_ctx);
386   if (ssl == nullptr) {
387     return nullptr;
388   }
389 
390 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
391   assert(SSL_is_quic(ssl));
392 #endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
393 
394   SSL_set_accept_state(ssl);
395 
396   auto config = get_config();
397   auto &quicconf = config->quic;
398 
399   if (quicconf.upstream.early_data) {
400 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
401     SSL_set_quic_early_data_enabled(ssl, 1);
402 #else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
403     SSL_set_early_data_enabled(ssl, 1);
404 #endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
405   }
406 
407   // Disable TLS session ticket if we don't have working ticket
408   // keys.
409   if (!worker_->get_ticket_keys()) {
410     SSL_set_options(ssl, SSL_OP_NO_TICKET);
411   }
412 
413   auto handler = std::make_unique<ClientHandler>(
414       worker_, faddr->fd, ssl, StringRef{host.data()},
415       StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
416 
417   auto upstream = std::make_unique<Http3Upstream>(handler.get());
418   if (upstream->init(faddr, remote_addr, local_addr, hd, odcid, token,
419                      tokenlen) != 0) {
420     return nullptr;
421   }
422 
423   handler->setup_http3_upstream(std::move(upstream));
424 
425   return handler.release();
426 }
427 
428 namespace {
generate_reserved_version(const Address & addr,uint32_t version)429 uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
430   uint32_t h = 0x811C9DC5u;
431   const uint8_t *p = reinterpret_cast<const uint8_t *>(&addr.su.sa);
432   const uint8_t *ep = p + addr.len;
433 
434   for (; p != ep; ++p) {
435     h ^= *p;
436     h *= 0x01000193u;
437   }
438 
439   version = htonl(version);
440   p = (const uint8_t *)&version;
441   ep = p + sizeof(version);
442 
443   for (; p != ep; ++p) {
444     h ^= *p;
445     h *= 0x01000193u;
446   }
447 
448   h &= 0xf0f0f0f0u;
449   h |= 0x0a0a0a0au;
450 
451   return h;
452 }
453 } // namespace
454 
send_retry(const UpstreamAddr * faddr,uint32_t version,const uint8_t * ini_dcid,size_t ini_dcidlen,const uint8_t * ini_scid,size_t ini_scidlen,const Address & remote_addr,const Address & local_addr,size_t max_pktlen)455 int QUICConnectionHandler::send_retry(
456     const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
457     size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
458     const Address &remote_addr, const Address &local_addr, size_t max_pktlen) {
459   std::array<char, NI_MAXHOST> host;
460   std::array<char, NI_MAXSERV> port;
461 
462   if (getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(), host.size(),
463                   port.data(), port.size(),
464                   NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
465     return -1;
466   }
467 
468   auto config = get_config();
469   auto &quicconf = config->quic;
470 
471   auto conn_handler = worker_->get_connection_handler();
472   auto &qkms = conn_handler->get_quic_keying_materials();
473   auto &qkm = qkms->keying_materials.front();
474 
475   ngtcp2_cid retry_scid;
476 
477   if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN,
478                                         quicconf.server_id.data(), qkm.id,
479                                         qkm.cid_encryption_key.data()) != 0) {
480     return -1;
481   }
482 
483   std::array<uint8_t, NGTCP2_CRYPTO_MAX_RETRY_TOKENLEN> token;
484   size_t tokenlen;
485 
486   ngtcp2_cid idcid, iscid;
487   ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
488   ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
489 
490   if (generate_retry_token(token.data(), tokenlen, version, &remote_addr.su.sa,
491                            remote_addr.len, retry_scid, idcid,
492                            qkm.secret.data(), qkm.secret.size()) != 0) {
493     return -1;
494   }
495 
496   std::vector<uint8_t> buf;
497   buf.resize(std::min(max_pktlen, static_cast<size_t>(256)));
498 
499   auto nwrite =
500       ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid,
501                                 &retry_scid, &idcid, token.data(), tokenlen);
502   if (nwrite < 0) {
503     LOG(ERROR) << "ngtcp2_crypto_write_retry: " << ngtcp2_strerror(nwrite);
504     return -1;
505   }
506 
507   buf.resize(nwrite);
508 
509   quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
510                    &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
511                    buf.data(), buf.size(), 0);
512 
513   if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr,
514                                          idcid) != 0) {
515     return -1;
516   }
517 
518   auto d =
519       static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS;
520 
521   if (LOG_ENABLED(INFO)) {
522     LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size()
523               << " bytes sentinel packet";
524   }
525 
526   auto cw = std::make_unique<CloseWait>(worker_, std::vector<ngtcp2_cid>{idcid},
527                                         std::move(buf), d);
528 
529   add_close_wait(cw.get());
530 
531   cw.release();
532 
533   return 0;
534 }
535 
send_version_negotiation(const UpstreamAddr * faddr,uint32_t version,const uint8_t * ini_dcid,size_t ini_dcidlen,const uint8_t * ini_scid,size_t ini_scidlen,const Address & remote_addr,const Address & local_addr)536 int QUICConnectionHandler::send_version_negotiation(
537     const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
538     size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
539     const Address &remote_addr, const Address &local_addr) {
540   std::array<uint32_t, 2> sv{
541       generate_reserved_version(remote_addr, version),
542       NGTCP2_PROTO_VER_V1,
543   };
544 
545   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
546 
547   uint8_t rand_byte;
548   util::random_bytes(&rand_byte, &rand_byte + 1, worker_->get_randgen());
549 
550   auto nwrite = ngtcp2_pkt_write_version_negotiation(
551       buf.data(), buf.size(), rand_byte, ini_scid, ini_scidlen, ini_dcid,
552       ini_dcidlen, sv.data(), sv.size());
553   if (nwrite < 0) {
554     LOG(ERROR) << "ngtcp2_pkt_write_version_negotiation: "
555                << ngtcp2_strerror(nwrite);
556     return -1;
557   }
558 
559   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
560                           &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
561                           buf.data(), nwrite, 0);
562 }
563 
send_stateless_reset(const UpstreamAddr * faddr,const uint8_t * dcid,size_t dcidlen,const Address & remote_addr,const Address & local_addr)564 int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
565                                                 const uint8_t *dcid,
566                                                 size_t dcidlen,
567                                                 const Address &remote_addr,
568                                                 const Address &local_addr) {
569   if (stateless_reset_bucket_ == 0) {
570     if (LOG_ENABLED(INFO)) {
571       LOG(INFO) << "Stateless Reset bucket has been depleted";
572     }
573 
574     return 0;
575   }
576 
577   --stateless_reset_bucket_;
578 
579   if (!ev_is_active(&stateless_reset_bucket_regen_timer_)) {
580     ev_timer_again(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
581   }
582 
583   int rv;
584   std::array<uint8_t, NGTCP2_STATELESS_RESET_TOKENLEN> token;
585   ngtcp2_cid cid;
586 
587   ngtcp2_cid_init(&cid, dcid, dcidlen);
588 
589   auto conn_handler = worker_->get_connection_handler();
590   auto &qkms = conn_handler->get_quic_keying_materials();
591   auto &qkm = qkms->keying_materials.front();
592 
593   rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
594                                            qkm.secret.size());
595   if (rv != 0) {
596     return -1;
597   }
598 
599   std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes;
600 
601   if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) {
602     return -1;
603   }
604 
605   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
606 
607   auto nwrite =
608       ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(),
609                                        rand_bytes.data(), rand_bytes.size());
610   if (nwrite < 0) {
611     LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: "
612                << ngtcp2_strerror(nwrite);
613     return -1;
614   }
615 
616   if (LOG_ENABLED(INFO)) {
617     LOG(INFO) << "Send stateless_reset to remote="
618               << util::to_numeric_addr(&remote_addr)
619               << " dcid=" << util::format_hex(dcid, dcidlen);
620   }
621 
622   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
623                           &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
624                           buf.data(), nwrite, 0);
625 }
626 
send_connection_close(const UpstreamAddr * faddr,uint32_t version,const ngtcp2_cid & ini_dcid,const ngtcp2_cid & ini_scid,const Address & remote_addr,const Address & local_addr,uint64_t error_code,size_t max_pktlen)627 int QUICConnectionHandler::send_connection_close(
628     const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid,
629     const ngtcp2_cid &ini_scid, const Address &remote_addr,
630     const Address &local_addr, uint64_t error_code, size_t max_pktlen) {
631   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
632 
633   max_pktlen = std::min(max_pktlen, buf.size());
634 
635   auto nwrite = ngtcp2_crypto_write_connection_close(
636       buf.data(), max_pktlen, version, &ini_scid, &ini_dcid, error_code,
637       nullptr, 0);
638   if (nwrite < 0) {
639     LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed";
640     return -1;
641   }
642 
643   if (LOG_ENABLED(INFO)) {
644     LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex
645               << error_code << log::dec
646               << " to remote=" << util::to_numeric_addr(&remote_addr)
647               << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen)
648               << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen);
649   }
650 
651   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
652                           &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
653                           buf.data(), nwrite, 0);
654 }
655 
add_connection_id(const ngtcp2_cid & cid,ClientHandler * handler)656 void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid,
657                                               ClientHandler *handler) {
658   connections_.emplace(cid, handler);
659 }
660 
remove_connection_id(const ngtcp2_cid & cid)661 void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) {
662   connections_.erase(cid);
663 }
664 
add_close_wait(CloseWait * cw)665 void QUICConnectionHandler::add_close_wait(CloseWait *cw) {
666   for (auto &cid : cw->scids) {
667     close_waits_.emplace(cid, cw);
668   }
669 }
670 
remove_close_wait(const CloseWait * cw)671 void QUICConnectionHandler::remove_close_wait(const CloseWait *cw) {
672   for (auto &cid : cw->scids) {
673     close_waits_.erase(cid);
674   }
675 }
676 
on_stateless_reset_bucket_regen()677 void QUICConnectionHandler::on_stateless_reset_bucket_regen() {
678   assert(stateless_reset_bucket_ < SHRPX_QUIC_STATELESS_RESET_BURST);
679 
680   if (++stateless_reset_bucket_ == SHRPX_QUIC_STATELESS_RESET_BURST) {
681     ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
682   }
683 }
684 
close_wait_timeoutcb(struct ev_loop * loop,ev_timer * w,int revents)685 static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w,
686                                  int revents) {
687   auto cw = static_cast<CloseWait *>(w->data);
688 
689   if (LOG_ENABLED(INFO)) {
690     LOG(INFO) << "close-wait period finished";
691   }
692 
693   auto quic_conn_handler = cw->worker->get_quic_connection_handler();
694   quic_conn_handler->remove_close_wait(cw);
695 
696   delete cw;
697 }
698 
CloseWait(Worker * worker,std::vector<ngtcp2_cid> scids,std::vector<uint8_t> pkt,ev_tstamp period)699 CloseWait::CloseWait(Worker *worker, std::vector<ngtcp2_cid> scids,
700                      std::vector<uint8_t> pkt, ev_tstamp period)
701     : worker{worker},
702       scids{std::move(scids)},
703       pkt{std::move(pkt)},
704       bytes_recv{0},
705       bytes_sent{0},
706       num_pkts_recv{0},
707       next_pkts_recv{1} {
708   ++worker->get_worker_stat()->num_close_waits;
709 
710   ev_timer_init(&timer, close_wait_timeoutcb, period, 0.);
711   timer.data = this;
712 
713   ev_timer_start(worker->get_loop(), &timer);
714 }
715 
~CloseWait()716 CloseWait::~CloseWait() {
717   auto loop = worker->get_loop();
718 
719   ev_timer_stop(loop, &timer);
720 
721   auto worker_stat = worker->get_worker_stat();
722   --worker_stat->num_close_waits;
723 
724   if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 &&
725       worker_stat->num_close_waits == 0) {
726     ev_break(loop);
727   }
728 }
729 
handle_packet(const UpstreamAddr * faddr,const Address & remote_addr,const Address & local_addr,const ngtcp2_pkt_info & pi,const uint8_t * data,size_t datalen)730 int CloseWait::handle_packet(const UpstreamAddr *faddr,
731                              const Address &remote_addr,
732                              const Address &local_addr,
733                              const ngtcp2_pkt_info &pi, const uint8_t *data,
734                              size_t datalen) {
735   if (pkt.empty()) {
736     return 0;
737   }
738 
739   ++num_pkts_recv;
740   bytes_recv += datalen;
741 
742   if (bytes_sent + pkt.size() > 3 * bytes_recv ||
743       next_pkts_recv > num_pkts_recv) {
744     return 0;
745   }
746 
747   if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
748                        &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
749                        pkt.data(), pkt.size(), 0) != 0) {
750     return -1;
751   }
752 
753   next_pkts_recv *= 2;
754   bytes_sent += pkt.size();
755 
756   return 0;
757 }
758 
759 } // namespace shrpx
760