• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * nghttp2 - HTTP/2 C Library
3  *
4  * Copyright (c) 2021 Tatsuhiro Tsujikawa
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining
7  * a copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sublicense, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice shall be
15  * included in all copies or substantial portions of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  */
25 #include "shrpx_quic_connection_handler.h"
26 
27 #include <openssl/rand.h>
28 
29 #include <ngtcp2/ngtcp2.h>
30 #include <ngtcp2/ngtcp2_crypto.h>
31 
32 #include "shrpx_worker.h"
33 #include "shrpx_client_handler.h"
34 #include "shrpx_log.h"
35 #include "shrpx_http3_upstream.h"
36 #include "shrpx_connection_handler.h"
37 
38 namespace shrpx {
39 
40 namespace {
stateless_reset_bucket_regen_timercb(struct ev_loop * loop,ev_timer * w,int revents)41 void stateless_reset_bucket_regen_timercb(struct ev_loop *loop, ev_timer *w,
42                                           int revents) {
43   auto quic_conn_handler = static_cast<QUICConnectionHandler *>(w->data);
44 
45   quic_conn_handler->on_stateless_reset_bucket_regen();
46 }
47 } // namespace
48 
QUICConnectionHandler(Worker * worker)49 QUICConnectionHandler::QUICConnectionHandler(Worker *worker)
50     : worker_{worker},
51       stateless_reset_bucket_{SHRPX_QUIC_STATELESS_RESET_BURST} {
52   ev_timer_init(&stateless_reset_bucket_regen_timer_,
53                 stateless_reset_bucket_regen_timercb, 0., 1.);
54   stateless_reset_bucket_regen_timer_.data = this;
55 }
56 
~QUICConnectionHandler()57 QUICConnectionHandler::~QUICConnectionHandler() {
58   ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
59 }
60 
handle_packet(const UpstreamAddr * faddr,const Address & remote_addr,const Address & local_addr,const ngtcp2_pkt_info & pi,const uint8_t * data,size_t datalen)61 int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
62                                          const Address &remote_addr,
63                                          const Address &local_addr,
64                                          const ngtcp2_pkt_info &pi,
65                                          const uint8_t *data, size_t datalen) {
66   int rv;
67   ngtcp2_version_cid vc;
68 
69   rv = ngtcp2_pkt_decode_version_cid(&vc, data, datalen, SHRPX_QUIC_SCIDLEN);
70   switch (rv) {
71   case 0:
72     break;
73   case NGTCP2_ERR_VERSION_NEGOTIATION:
74     send_version_negotiation(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
75                              vc.scidlen, remote_addr, local_addr);
76 
77     return 0;
78   default:
79     return 0;
80   }
81 
82   auto config = get_config();
83 
84   ngtcp2_cid dcid_key;
85   ngtcp2_cid_init(&dcid_key, vc.dcid, vc.dcidlen);
86 
87   auto conn_handler = worker_->get_connection_handler();
88 
89   ClientHandler *handler;
90 
91   auto &quicconf = config->quic;
92 
93   auto it = connections_.find(dcid_key);
94   if (it == std::end(connections_)) {
95     auto cwit = close_waits_.find(dcid_key);
96     if (cwit != std::end(close_waits_)) {
97       auto cw = (*cwit).second;
98 
99       cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
100 
101       return 0;
102     }
103 
104     if (data[0] & 0x80) {
105       if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr,
106                                              dcid_key) != 0) {
107         return 0;
108       }
109 
110       it = connections_.find(dcid_key);
111       if (it == std::end(connections_)) {
112         auto cwit = close_waits_.find(dcid_key);
113         if (cwit != std::end(close_waits_)) {
114           auto cw = (*cwit).second;
115 
116           cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
117 
118           return 0;
119         }
120       }
121     }
122   }
123 
124   if (it == std::end(connections_)) {
125     std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
126 
127     auto &qkms = conn_handler->get_quic_keying_materials();
128     const QUICKeyingMaterial *qkm = nullptr;
129 
130     if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) {
131       qkm = select_quic_keying_material(
132           *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK);
133 
134       if (decrypt_quic_connection_id(decrypted_dcid.data(),
135                                      vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
136                                      qkm->cid_encryption_key.data()) != 0) {
137         return 0;
138       }
139 
140       if (qkm != &qkms->keying_materials.front() ||
141           !std::equal(std::begin(decrypted_dcid),
142                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
143                       worker_->get_cid_prefix())) {
144         auto quic_lwp =
145             conn_handler->match_quic_lingering_worker_process_cid_prefix(
146                 decrypted_dcid.data(), decrypted_dcid.size());
147         if (quic_lwp) {
148           if (conn_handler->forward_quic_packet_to_lingering_worker_process(
149                   quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) {
150             return 0;
151           }
152 
153           return 0;
154         }
155       }
156     }
157 
158     // new connection
159 
160     auto &upstreamconf = config->conn.upstream;
161     if (worker_->get_worker_stat()->num_connections >=
162         upstreamconf.worker_connections) {
163       if (LOG_ENABLED(INFO)) {
164         LOG(INFO) << "Too many connections >="
165                   << upstreamconf.worker_connections;
166       }
167 
168       return 0;
169     }
170 
171     ngtcp2_pkt_hd hd;
172     ngtcp2_cid odcid, *podcid = nullptr;
173     const uint8_t *token = nullptr;
174     size_t tokenlen = 0;
175     ngtcp2_token_type token_type = NGTCP2_TOKEN_TYPE_UNKNOWN;
176 
177     switch (ngtcp2_accept(&hd, data, datalen)) {
178     case 0: {
179       // If we get Initial and it has the CID prefix of this worker,
180       // it is likely that client is intentionally use the prefix.
181       // Just drop it.
182       if (vc.dcidlen == SHRPX_QUIC_SCIDLEN) {
183         if (qkm != &qkms->keying_materials.front()) {
184           qkm = &qkms->keying_materials.front();
185 
186           if (decrypt_quic_connection_id(decrypted_dcid.data(),
187                                          vc.dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
188                                          qkm->cid_encryption_key.data()) != 0) {
189             return 0;
190           }
191         }
192 
193         if (std::equal(std::begin(decrypted_dcid),
194                        std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
195                        worker_->get_cid_prefix())) {
196           return 0;
197         }
198       }
199 
200       if (worker_->get_graceful_shutdown()) {
201         send_connection_close(faddr, hd.version, hd.dcid, hd.scid, remote_addr,
202                               local_addr, NGTCP2_CONNECTION_REFUSED,
203                               datalen * 3);
204         return 0;
205       }
206 
207       if (hd.tokenlen == 0) {
208         if (quicconf.upstream.require_token) {
209           send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
210                      vc.scidlen, remote_addr, local_addr, datalen * 3);
211 
212           return 0;
213         }
214 
215         break;
216       }
217 
218       switch (hd.token[0]) {
219       case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY: {
220         if (vc.dcidlen != SHRPX_QUIC_SCIDLEN) {
221           // Initial packets with Retry token must have DCID chosen by
222           // server.
223           return 0;
224         }
225 
226         auto qkm = select_quic_keying_material(
227             *qkms.get(), vc.dcid[0] & SHRPX_QUIC_DCID_KM_ID_MASK);
228 
229         if (verify_retry_token(odcid, hd.token, hd.tokenlen, hd.version,
230                                hd.dcid, &remote_addr.su.sa, remote_addr.len,
231                                qkm->secret.data(), qkm->secret.size()) != 0) {
232           if (LOG_ENABLED(INFO)) {
233             LOG(INFO) << "Failed to validate Retry token from remote="
234                       << util::to_numeric_addr(&remote_addr);
235           }
236 
237           // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE
238           // with INVALID_TOKEN.
239           send_connection_close(faddr, hd.version, hd.dcid, hd.scid,
240                                 remote_addr, local_addr, NGTCP2_INVALID_TOKEN,
241                                 datalen * 3);
242           return 0;
243         }
244 
245         if (LOG_ENABLED(INFO)) {
246           LOG(INFO) << "Successfully validated Retry token from remote="
247                     << util::to_numeric_addr(&remote_addr);
248         }
249 
250         podcid = &odcid;
251         token = hd.token;
252         tokenlen = hd.tokenlen;
253         token_type = NGTCP2_TOKEN_TYPE_RETRY;
254 
255         break;
256       }
257       case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR: {
258         // If a token is a regular token, it must be at least
259         // NGTCP2_MIN_INITIAL_DCIDLEN bytes long.
260         if (vc.dcidlen < NGTCP2_MIN_INITIAL_DCIDLEN) {
261           return 0;
262         }
263 
264         if (hd.tokenlen != NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN + 1) {
265           if (LOG_ENABLED(INFO)) {
266             LOG(INFO) << "Failed to validate token from remote="
267                       << util::to_numeric_addr(&remote_addr);
268           }
269 
270           if (quicconf.upstream.require_token) {
271             send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
272                        vc.scidlen, remote_addr, local_addr, datalen * 3);
273 
274             return 0;
275           }
276 
277           break;
278         }
279 
280         auto qkm = select_quic_keying_material(
281             *qkms.get(), hd.token[NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN]);
282 
283         if (verify_token(hd.token, hd.tokenlen - 1, &remote_addr.su.sa,
284                          remote_addr.len, qkm->secret.data(),
285                          qkm->secret.size()) != 0) {
286           if (LOG_ENABLED(INFO)) {
287             LOG(INFO) << "Failed to validate token from remote="
288                       << util::to_numeric_addr(&remote_addr);
289           }
290 
291           if (quicconf.upstream.require_token) {
292             send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
293                        vc.scidlen, remote_addr, local_addr, datalen * 3);
294 
295             return 0;
296           }
297 
298           break;
299         }
300 
301         if (LOG_ENABLED(INFO)) {
302           LOG(INFO) << "Successfully validated token from remote="
303                     << util::to_numeric_addr(&remote_addr);
304         }
305 
306         token = hd.token;
307         tokenlen = hd.tokenlen;
308         token_type = NGTCP2_TOKEN_TYPE_NEW_TOKEN;
309 
310         break;
311       }
312       default:
313         if (quicconf.upstream.require_token) {
314           send_retry(faddr, vc.version, vc.dcid, vc.dcidlen, vc.scid,
315                      vc.scidlen, remote_addr, local_addr, datalen * 3);
316 
317           return 0;
318         }
319 
320         break;
321       }
322 
323       break;
324     }
325     default:
326       if (!config->single_thread && !(data[0] & 0x80) &&
327           vc.dcidlen == SHRPX_QUIC_SCIDLEN &&
328           !std::equal(std::begin(decrypted_dcid),
329                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
330                       worker_->get_cid_prefix())) {
331         if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr,
332                                               pi, decrypted_dcid.data(), data,
333                                               datalen) == 0) {
334           return 0;
335         }
336       }
337 
338       if (!(data[0] & 0x80)) {
339         // TODO Must be rate limited
340         send_stateless_reset(faddr, vc.dcid, vc.dcidlen, remote_addr,
341                              local_addr);
342       }
343 
344       return 0;
345     }
346 
347     handler = handle_new_connection(faddr, remote_addr, local_addr, hd, podcid,
348                                     token, tokenlen, token_type);
349     if (handler == nullptr) {
350       return 0;
351     }
352   } else {
353     handler = (*it).second;
354   }
355 
356   if (handler->read_quic(faddr, remote_addr, local_addr, pi, data, datalen) !=
357       0) {
358     delete handler;
359     return 0;
360   }
361 
362   handler->signal_write();
363 
364   return 0;
365 }
366 
handle_new_connection(const UpstreamAddr * faddr,const Address & remote_addr,const Address & local_addr,const ngtcp2_pkt_hd & hd,const ngtcp2_cid * odcid,const uint8_t * token,size_t tokenlen,ngtcp2_token_type token_type)367 ClientHandler *QUICConnectionHandler::handle_new_connection(
368     const UpstreamAddr *faddr, const Address &remote_addr,
369     const Address &local_addr, const ngtcp2_pkt_hd &hd, const ngtcp2_cid *odcid,
370     const uint8_t *token, size_t tokenlen, ngtcp2_token_type token_type) {
371   std::array<char, NI_MAXHOST> host;
372   std::array<char, NI_MAXSERV> service;
373   int rv;
374 
375   rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
376                    host.size(), service.data(), service.size(),
377                    NI_NUMERICHOST | NI_NUMERICSERV);
378   if (rv != 0) {
379     LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
380 
381     return nullptr;
382   }
383 
384   auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
385 
386   assert(ssl_ctx);
387 
388   auto ssl = tls::create_ssl(ssl_ctx);
389   if (ssl == nullptr) {
390     return nullptr;
391   }
392 
393 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
394   assert(SSL_is_quic(ssl));
395 #endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
396 
397   SSL_set_accept_state(ssl);
398 
399   auto config = get_config();
400   auto &quicconf = config->quic;
401 
402   if (quicconf.upstream.early_data) {
403 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
404     SSL_set_quic_early_data_enabled(ssl, 1);
405 #else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
406     SSL_set_early_data_enabled(ssl, 1);
407 #endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
408   }
409 
410   // Disable TLS session ticket if we don't have working ticket
411   // keys.
412   if (!worker_->get_ticket_keys()) {
413     SSL_set_options(ssl, SSL_OP_NO_TICKET);
414   }
415 
416   auto handler = std::make_unique<ClientHandler>(
417       worker_, faddr->fd, ssl, StringRef{host.data()},
418       StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
419 
420   auto upstream = std::make_unique<Http3Upstream>(handler.get());
421   if (upstream->init(faddr, remote_addr, local_addr, hd, odcid, token, tokenlen,
422                      token_type) != 0) {
423     return nullptr;
424   }
425 
426   handler->setup_http3_upstream(std::move(upstream));
427 
428   return handler.release();
429 }
430 
431 namespace {
generate_reserved_version(const Address & addr,uint32_t version)432 uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
433   uint32_t h = 0x811C9DC5u;
434   const uint8_t *p = reinterpret_cast<const uint8_t *>(&addr.su.sa);
435   const uint8_t *ep = p + addr.len;
436 
437   for (; p != ep; ++p) {
438     h ^= *p;
439     h *= 0x01000193u;
440   }
441 
442   version = htonl(version);
443   p = (const uint8_t *)&version;
444   ep = p + sizeof(version);
445 
446   for (; p != ep; ++p) {
447     h ^= *p;
448     h *= 0x01000193u;
449   }
450 
451   h &= 0xf0f0f0f0u;
452   h |= 0x0a0a0a0au;
453 
454   return h;
455 }
456 } // namespace
457 
send_retry(const UpstreamAddr * faddr,uint32_t version,const uint8_t * ini_dcid,size_t ini_dcidlen,const uint8_t * ini_scid,size_t ini_scidlen,const Address & remote_addr,const Address & local_addr,size_t max_pktlen)458 int QUICConnectionHandler::send_retry(
459     const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
460     size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
461     const Address &remote_addr, const Address &local_addr, size_t max_pktlen) {
462   std::array<char, NI_MAXHOST> host;
463   std::array<char, NI_MAXSERV> port;
464 
465   if (getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(), host.size(),
466                   port.data(), port.size(),
467                   NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
468     return -1;
469   }
470 
471   auto config = get_config();
472   auto &quicconf = config->quic;
473 
474   auto conn_handler = worker_->get_connection_handler();
475   auto &qkms = conn_handler->get_quic_keying_materials();
476   auto &qkm = qkms->keying_materials.front();
477 
478   ngtcp2_cid retry_scid;
479 
480   if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN,
481                                         quicconf.server_id.data(), qkm.id,
482                                         qkm.cid_encryption_key.data()) != 0) {
483     return -1;
484   }
485 
486   std::array<uint8_t, NGTCP2_CRYPTO_MAX_RETRY_TOKENLEN> token;
487   size_t tokenlen;
488 
489   ngtcp2_cid idcid, iscid;
490   ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
491   ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
492 
493   if (generate_retry_token(token.data(), tokenlen, version, &remote_addr.su.sa,
494                            remote_addr.len, retry_scid, idcid,
495                            qkm.secret.data(), qkm.secret.size()) != 0) {
496     return -1;
497   }
498 
499   std::vector<uint8_t> buf;
500   buf.resize(std::min(max_pktlen, static_cast<size_t>(256)));
501 
502   auto nwrite =
503       ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid,
504                                 &retry_scid, &idcid, token.data(), tokenlen);
505   if (nwrite < 0) {
506     LOG(ERROR) << "ngtcp2_crypto_write_retry: " << ngtcp2_strerror(nwrite);
507     return -1;
508   }
509 
510   buf.resize(nwrite);
511 
512   quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
513                    &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
514                    buf.data(), buf.size(), 0);
515 
516   if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr,
517                                          idcid) != 0) {
518     return -1;
519   }
520 
521   auto d =
522       static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS;
523 
524   if (LOG_ENABLED(INFO)) {
525     LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size()
526               << " bytes sentinel packet";
527   }
528 
529   auto cw = std::make_unique<CloseWait>(worker_, std::vector<ngtcp2_cid>{idcid},
530                                         std::move(buf), d);
531 
532   add_close_wait(cw.release());
533 
534   return 0;
535 }
536 
send_version_negotiation(const UpstreamAddr * faddr,uint32_t version,const uint8_t * ini_dcid,size_t ini_dcidlen,const uint8_t * ini_scid,size_t ini_scidlen,const Address & remote_addr,const Address & local_addr)537 int QUICConnectionHandler::send_version_negotiation(
538     const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
539     size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
540     const Address &remote_addr, const Address &local_addr) {
541   std::array<uint32_t, 2> sv{
542       generate_reserved_version(remote_addr, version),
543       NGTCP2_PROTO_VER_V1,
544   };
545 
546   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
547 
548   uint8_t rand_byte;
549   util::random_bytes(&rand_byte, &rand_byte + 1, worker_->get_randgen());
550 
551   auto nwrite = ngtcp2_pkt_write_version_negotiation(
552       buf.data(), buf.size(), rand_byte, ini_scid, ini_scidlen, ini_dcid,
553       ini_dcidlen, sv.data(), sv.size());
554   if (nwrite < 0) {
555     LOG(ERROR) << "ngtcp2_pkt_write_version_negotiation: "
556                << ngtcp2_strerror(nwrite);
557     return -1;
558   }
559 
560   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
561                           &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
562                           buf.data(), nwrite, 0);
563 }
564 
send_stateless_reset(const UpstreamAddr * faddr,const uint8_t * dcid,size_t dcidlen,const Address & remote_addr,const Address & local_addr)565 int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
566                                                 const uint8_t *dcid,
567                                                 size_t dcidlen,
568                                                 const Address &remote_addr,
569                                                 const Address &local_addr) {
570   if (stateless_reset_bucket_ == 0) {
571     if (LOG_ENABLED(INFO)) {
572       LOG(INFO) << "Stateless Reset bucket has been depleted";
573     }
574 
575     return 0;
576   }
577 
578   --stateless_reset_bucket_;
579 
580   if (!ev_is_active(&stateless_reset_bucket_regen_timer_)) {
581     ev_timer_again(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
582   }
583 
584   int rv;
585   std::array<uint8_t, NGTCP2_STATELESS_RESET_TOKENLEN> token;
586   ngtcp2_cid cid;
587 
588   ngtcp2_cid_init(&cid, dcid, dcidlen);
589 
590   auto conn_handler = worker_->get_connection_handler();
591   auto &qkms = conn_handler->get_quic_keying_materials();
592   auto &qkm = qkms->keying_materials.front();
593 
594   rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
595                                            qkm.secret.size());
596   if (rv != 0) {
597     return -1;
598   }
599 
600   std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes;
601 
602   if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) {
603     return -1;
604   }
605 
606   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
607 
608   auto nwrite =
609       ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(),
610                                        rand_bytes.data(), rand_bytes.size());
611   if (nwrite < 0) {
612     LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: "
613                << ngtcp2_strerror(nwrite);
614     return -1;
615   }
616 
617   if (LOG_ENABLED(INFO)) {
618     LOG(INFO) << "Send stateless_reset to remote="
619               << util::to_numeric_addr(&remote_addr)
620               << " dcid=" << util::format_hex(dcid, dcidlen);
621   }
622 
623   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
624                           &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
625                           buf.data(), nwrite, 0);
626 }
627 
send_connection_close(const UpstreamAddr * faddr,uint32_t version,const ngtcp2_cid & ini_dcid,const ngtcp2_cid & ini_scid,const Address & remote_addr,const Address & local_addr,uint64_t error_code,size_t max_pktlen)628 int QUICConnectionHandler::send_connection_close(
629     const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid,
630     const ngtcp2_cid &ini_scid, const Address &remote_addr,
631     const Address &local_addr, uint64_t error_code, size_t max_pktlen) {
632   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
633 
634   max_pktlen = std::min(max_pktlen, buf.size());
635 
636   auto nwrite = ngtcp2_crypto_write_connection_close(
637       buf.data(), max_pktlen, version, &ini_scid, &ini_dcid, error_code,
638       nullptr, 0);
639   if (nwrite < 0) {
640     LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed";
641     return -1;
642   }
643 
644   if (LOG_ENABLED(INFO)) {
645     LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex
646               << error_code << log::dec
647               << " to remote=" << util::to_numeric_addr(&remote_addr)
648               << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen)
649               << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen);
650   }
651 
652   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
653                           &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
654                           buf.data(), nwrite, 0);
655 }
656 
add_connection_id(const ngtcp2_cid & cid,ClientHandler * handler)657 void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid,
658                                               ClientHandler *handler) {
659   connections_.emplace(cid, handler);
660 }
661 
remove_connection_id(const ngtcp2_cid & cid)662 void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) {
663   connections_.erase(cid);
664 }
665 
add_close_wait(CloseWait * cw)666 void QUICConnectionHandler::add_close_wait(CloseWait *cw) {
667   for (auto &cid : cw->scids) {
668     close_waits_.emplace(cid, cw);
669   }
670 }
671 
remove_close_wait(const CloseWait * cw)672 void QUICConnectionHandler::remove_close_wait(const CloseWait *cw) {
673   for (auto &cid : cw->scids) {
674     close_waits_.erase(cid);
675   }
676 }
677 
on_stateless_reset_bucket_regen()678 void QUICConnectionHandler::on_stateless_reset_bucket_regen() {
679   assert(stateless_reset_bucket_ < SHRPX_QUIC_STATELESS_RESET_BURST);
680 
681   if (++stateless_reset_bucket_ == SHRPX_QUIC_STATELESS_RESET_BURST) {
682     ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
683   }
684 }
685 
close_wait_timeoutcb(struct ev_loop * loop,ev_timer * w,int revents)686 static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w,
687                                  int revents) {
688   auto cw = static_cast<CloseWait *>(w->data);
689 
690   if (LOG_ENABLED(INFO)) {
691     LOG(INFO) << "close-wait period finished";
692   }
693 
694   auto quic_conn_handler = cw->worker->get_quic_connection_handler();
695   quic_conn_handler->remove_close_wait(cw);
696 
697   delete cw;
698 }
699 
CloseWait(Worker * worker,std::vector<ngtcp2_cid> scids,std::vector<uint8_t> pkt,ev_tstamp period)700 CloseWait::CloseWait(Worker *worker, std::vector<ngtcp2_cid> scids,
701                      std::vector<uint8_t> pkt, ev_tstamp period)
702     : worker{worker},
703       scids{std::move(scids)},
704       pkt{std::move(pkt)},
705       bytes_recv{0},
706       bytes_sent{0},
707       num_pkts_recv{0},
708       next_pkts_recv{1} {
709   ++worker->get_worker_stat()->num_close_waits;
710 
711   ev_timer_init(&timer, close_wait_timeoutcb, period, 0.);
712   timer.data = this;
713 
714   ev_timer_start(worker->get_loop(), &timer);
715 }
716 
~CloseWait()717 CloseWait::~CloseWait() {
718   auto loop = worker->get_loop();
719 
720   ev_timer_stop(loop, &timer);
721 
722   auto worker_stat = worker->get_worker_stat();
723   --worker_stat->num_close_waits;
724 
725   if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 &&
726       worker_stat->num_close_waits == 0) {
727     ev_break(loop);
728   }
729 }
730 
handle_packet(const UpstreamAddr * faddr,const Address & remote_addr,const Address & local_addr,const ngtcp2_pkt_info & pi,const uint8_t * data,size_t datalen)731 int CloseWait::handle_packet(const UpstreamAddr *faddr,
732                              const Address &remote_addr,
733                              const Address &local_addr,
734                              const ngtcp2_pkt_info &pi, const uint8_t *data,
735                              size_t datalen) {
736   if (pkt.empty()) {
737     return 0;
738   }
739 
740   ++num_pkts_recv;
741   bytes_recv += datalen;
742 
743   if (bytes_sent + pkt.size() > 3 * bytes_recv ||
744       next_pkts_recv > num_pkts_recv) {
745     return 0;
746   }
747 
748   if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
749                        &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
750                        pkt.data(), pkt.size(), 0) != 0) {
751     return -1;
752   }
753 
754   next_pkts_recv *= 2;
755   bytes_sent += pkt.size();
756 
757   return 0;
758 }
759 
760 } // namespace shrpx
761