1 /* 2 * nghttp2 - HTTP/2 C Library 3 * 4 * Copyright (c) 2012 Tatsuhiro Tsujikawa 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining 7 * a copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sublicense, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice shall be 15 * included in all copies or substantial portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 */ 25 #ifndef SHRPX_CONNECTION_HANDLER_H 26 #define SHRPX_CONNECTION_HANDLER_H 27 28 #include "shrpx.h" 29 30 #include <sys/types.h> 31 #ifdef HAVE_SYS_SOCKET_H 32 # include <sys/socket.h> 33 #endif // HAVE_SYS_SOCKET_H 34 35 #include <mutex> 36 #include <memory> 37 #include <vector> 38 #include <random> 39 #ifndef NOTHREADS 40 # include <future> 41 #endif // NOTHREADS 42 43 #ifdef HAVE_LIBBPF 44 # include <bpf/libbpf.h> 45 #endif // HAVE_LIBBPF 46 47 #include "ssl_compat.h" 48 49 #ifdef NGHTTP2_OPENSSL_IS_WOLFSSL 50 # include <wolfssl/options.h> 51 # include <wolfssl/openssl/ssl.h> 52 #else // !NGHTTP2_OPENSSL_IS_WOLFSSL 53 # include <openssl/ssl.h> 54 #endif // !NGHTTP2_OPENSSL_IS_WOLFSSL 55 56 #include <ev.h> 57 58 #ifdef HAVE_NEVERBLEED 59 # include <neverbleed.h> 60 #endif // HAVE_NEVERBLEED 61 62 #include "shrpx_downstream_connection_pool.h" 63 #include "shrpx_config.h" 64 #include "shrpx_exec.h" 65 66 namespace shrpx { 67 68 class Http2Session; 69 class ConnectBlocker; 70 class AcceptHandler; 71 class Worker; 72 struct WorkerStat; 73 struct TicketKeys; 74 class MemcachedDispatcher; 75 struct UpstreamAddr; 76 77 namespace tls { 78 79 class CertLookupTree; 80 81 } // namespace tls 82 83 struct OCSPUpdateContext { 84 // ocsp response buffer 85 std::vector<uint8_t> resp; 86 // Process running fetch-ocsp-response script 87 Process proc; 88 // index to ConnectionHandler::all_ssl_ctx_, which points to next 89 // SSL_CTX to update ocsp response cache. 90 size_t next; 91 ev_child chldev; 92 ev_io rev; 93 // errno encountered while processing response 94 int error; 95 }; 96 97 // SerialEvent is an event sent from Worker thread. 98 enum class SerialEventType { 99 NONE, 100 REPLACE_DOWNSTREAM, 101 }; 102 103 struct SerialEvent { 104 // ctor for event uses DownstreamConfig SerialEventSerialEvent105 SerialEvent(SerialEventType type, 106 const std::shared_ptr<DownstreamConfig> &downstreamconf) 107 : type(type), downstreamconf(downstreamconf) {} 108 109 SerialEventType type; 110 std::shared_ptr<DownstreamConfig> downstreamconf; 111 }; 112 113 #ifdef ENABLE_HTTP3 114 # ifdef HAVE_LIBBPF 115 struct BPFRef { 116 bpf_object *obj; 117 bpf_map *reuseport_array; 118 bpf_map *worker_id_map; 119 }; 120 # endif // HAVE_LIBBPF 121 122 // QUIC IPC message type. 123 enum class QUICIPCType { 124 NONE, 125 // Send forwarded QUIC UDP datagram and its metadata. 126 DGRAM_FORWARD, 127 }; 128 129 // WorkerProcesses which are in graceful shutdown period. 130 struct QUICLingeringWorkerProcess { QUICLingeringWorkerProcessQUICLingeringWorkerProcess131 QUICLingeringWorkerProcess(std::vector<WorkerID> worker_ids, int quic_ipc_fd) 132 : worker_ids{std::move(worker_ids)}, quic_ipc_fd{quic_ipc_fd} {} 133 134 std::vector<WorkerID> worker_ids; 135 // Socket to send QUIC IPC message to this worker process. 136 int quic_ipc_fd; 137 }; 138 #endif // ENABLE_HTTP3 139 140 class ConnectionHandler { 141 public: 142 ConnectionHandler(struct ev_loop *loop, std::mt19937 &gen); 143 ~ConnectionHandler(); 144 int handle_connection(int fd, sockaddr *addr, int addrlen, 145 const UpstreamAddr *faddr); 146 // Creates Worker object for single threaded configuration. 147 int create_single_worker(); 148 // Creates |num| Worker objects for multi threaded configuration. 149 // The |num| must be strictly more than 1. 150 int create_worker_thread(size_t num); 151 void 152 set_ticket_keys_to_worker(const std::shared_ptr<TicketKeys> &ticket_keys); 153 void worker_reopen_log_files(); 154 void set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys); 155 const std::shared_ptr<TicketKeys> &get_ticket_keys() const; 156 struct ev_loop *get_loop() const; 157 Worker *get_single_worker() const; 158 void add_acceptor(std::unique_ptr<AcceptHandler> h); 159 void delete_acceptor(); 160 void enable_acceptor(); 161 void disable_acceptor(); 162 void sleep_acceptor(ev_tstamp t); 163 void accept_pending_connection(); 164 void graceful_shutdown_worker(); 165 void set_graceful_shutdown(bool f); 166 bool get_graceful_shutdown() const; 167 void join_worker(); 168 169 // Cancels ocsp update process 170 void cancel_ocsp_update(); 171 // Starts ocsp update for certificate |cert_file|. 172 int start_ocsp_update(const char *cert_file); 173 // Reads incoming data from ocsp update process 174 void read_ocsp_chunk(); 175 // Handles the completion of one ocsp update 176 void handle_ocsp_complete(); 177 // Resets ocsp_; 178 void reset_ocsp(); 179 // Proceeds to the next certificate's ocsp update. If all 180 // certificates' ocsp update has been done, schedule next ocsp 181 // update. 182 void proceed_next_cert_ocsp(); 183 184 void set_tls_ticket_key_memcached_dispatcher( 185 std::unique_ptr<MemcachedDispatcher> dispatcher); 186 187 MemcachedDispatcher *get_tls_ticket_key_memcached_dispatcher() const; 188 void on_tls_ticket_key_network_error(ev_timer *w); 189 void on_tls_ticket_key_not_found(ev_timer *w); 190 void 191 on_tls_ticket_key_get_success(const std::shared_ptr<TicketKeys> &ticket_keys, 192 ev_timer *w); 193 void schedule_next_tls_ticket_key_memcached_get(ev_timer *w); 194 SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx(); 195 // Returns the SSL_CTX at all_ssl_ctx_[idx]. This does not perform 196 // array bound checking. 197 SSL_CTX *get_ssl_ctx(size_t idx) const; 198 199 const std::vector<SSL_CTX *> &get_indexed_ssl_ctx(size_t idx) const; 200 #ifdef ENABLE_HTTP3 201 const std::vector<SSL_CTX *> &get_quic_indexed_ssl_ctx(size_t idx) const; 202 203 int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr, 204 const Address &local_addr, const ngtcp2_pkt_info &pi, 205 const WorkerID &wid, std::span<const uint8_t> data); 206 207 void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms); 208 const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const; 209 210 void set_worker_ids(std::vector<WorkerID> worker_ids); 211 Worker *find_worker(const WorkerID &wid) const; 212 213 void set_quic_lingering_worker_processes( 214 const std::vector<QUICLingeringWorkerProcess> &quic_lwps); 215 216 // Return matching QUICLingeringWorkerProcess which has a Worker ID 217 // such that |dcid| starts with it. If no such 218 // QUICLingeringWorkerProcess, it returns nullptr. 219 QUICLingeringWorkerProcess * 220 match_quic_lingering_worker_process_worker_id(const WorkerID &wid); 221 222 int forward_quic_packet_to_lingering_worker_process( 223 QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, 224 const Address &local_addr, const ngtcp2_pkt_info &pi, 225 std::span<const uint8_t> data); 226 227 void set_quic_ipc_fd(int fd); 228 229 int quic_ipc_read(); 230 231 # ifdef HAVE_LIBBPF 232 std::vector<BPFRef> &get_quic_bpf_refs(); 233 void unload_bpf_objects(); 234 # endif // HAVE_LIBBPF 235 #endif // ENABLE_HTTP3 236 237 #ifdef HAVE_NEVERBLEED 238 void set_neverbleed(neverbleed_t *nb); 239 #endif // HAVE_NEVERBLEED 240 241 // Send SerialEvent SerialEventType::REPLACE_DOWNSTREAM to this 242 // object. 243 void send_replace_downstream( 244 const std::shared_ptr<DownstreamConfig> &downstreamconf); 245 // Internal function to send |ev| to this object. 246 void send_serial_event(SerialEvent ev); 247 // Handles SerialEvents received. 248 void handle_serial_event(); 249 // Sends WorkerEvent to make them replace downstream. 250 void 251 worker_replace_downstream(std::shared_ptr<DownstreamConfig> downstreamconf); 252 253 void set_enable_acceptor_on_ocsp_completion(bool f); 254 255 private: 256 // Stores all SSL_CTX objects. 257 std::vector<SSL_CTX *> all_ssl_ctx_; 258 // Stores all SSL_CTX objects in a way that its index is stored in 259 // cert_tree. The SSL_CTXs stored in the same index share the same 260 // hostname, but could have different signature algorithm. The 261 // selection among them are performed by hostname presented by SNI, 262 // and signature algorithm presented by client. 263 std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_; 264 #ifdef ENABLE_HTTP3 265 std::vector<WorkerID> worker_ids_; 266 std::vector<WorkerID> lingering_worker_ids_; 267 int quic_ipc_fd_; 268 std::vector<QUICLingeringWorkerProcess> quic_lingering_worker_processes_; 269 # ifdef HAVE_LIBBPF 270 std::vector<BPFRef> quic_bpf_refs_; 271 # endif // HAVE_LIBBPF 272 std::shared_ptr<QUICKeyingMaterials> quic_keying_materials_; 273 std::vector<SSL_CTX *> quic_all_ssl_ctx_; 274 std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_; 275 #endif // ENABLE_HTTP3 276 OCSPUpdateContext ocsp_; 277 std::mt19937 &gen_; 278 // ev_loop for each worker 279 std::vector<struct ev_loop *> worker_loops_; 280 // Worker instances when multi threaded mode (-nN, N >= 2) is used. 281 // If at least one frontend enables API request, we allocate 1 282 // additional worker dedicated to API request . 283 std::vector<std::unique_ptr<Worker>> workers_; 284 // mutex for serial event resive buffer handling 285 std::mutex serial_event_mu_; 286 // SerialEvent receive buffer 287 std::vector<SerialEvent> serial_events_; 288 // Worker instance used when single threaded mode (-n1) is used. 289 // Otherwise, nullptr and workers_ has instances of Worker instead. 290 std::unique_ptr<Worker> single_worker_; 291 std::unique_ptr<tls::CertLookupTree> cert_tree_; 292 #ifdef ENABLE_HTTP3 293 std::unique_ptr<tls::CertLookupTree> quic_cert_tree_; 294 #endif // ENABLE_HTTP3 295 std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_; 296 // Current TLS session ticket keys. Note that TLS connection does 297 // not refer to this field directly. They use TicketKeys object in 298 // Worker object. 299 std::shared_ptr<TicketKeys> ticket_keys_; 300 struct ev_loop *loop_; 301 std::vector<std::unique_ptr<AcceptHandler>> acceptors_; 302 #ifdef HAVE_NEVERBLEED 303 neverbleed_t *nb_; 304 #endif // HAVE_NEVERBLEED 305 ev_timer disable_acceptor_timer_; 306 ev_timer ocsp_timer_; 307 ev_async thread_join_asyncev_; 308 ev_async serial_event_asyncev_; 309 #ifndef NOTHREADS 310 std::future<void> thread_join_fut_; 311 #endif // NOTHREADS 312 size_t tls_ticket_key_memcached_get_retry_count_; 313 size_t tls_ticket_key_memcached_fail_count_; 314 unsigned int worker_round_robin_cnt_; 315 bool graceful_shutdown_; 316 // true if acceptors should be enabled after the initial ocsp update 317 // has finished. 318 bool enable_acceptor_on_ocsp_completion_; 319 }; 320 321 } // namespace shrpx 322 323 #endif // SHRPX_CONNECTION_HANDLER_H 324