1 /* 2 * nghttp2 - HTTP/2 C Library 3 * 4 * Copyright (c) 2012 Tatsuhiro Tsujikawa 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining 7 * a copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sublicense, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice shall be 15 * included in all copies or substantial portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 */ 25 #ifndef SHRPX_CONNECTION_HANDLER_H 26 #define SHRPX_CONNECTION_HANDLER_H 27 28 #include "shrpx.h" 29 30 #include <sys/types.h> 31 #ifdef HAVE_SYS_SOCKET_H 32 # include <sys/socket.h> 33 #endif // HAVE_SYS_SOCKET_H 34 35 #include <mutex> 36 #include <memory> 37 #include <vector> 38 #include <random> 39 #ifndef NOTHREADS 40 # include <future> 41 #endif // NOTHREADS 42 43 #ifdef HAVE_LIBBPF 44 # include <bpf/libbpf.h> 45 #endif // HAVE_LIBBPF 46 47 #include <openssl/ssl.h> 48 49 #include <ev.h> 50 51 #ifdef HAVE_NEVERBLEED 52 # include <neverbleed.h> 53 #endif // HAVE_NEVERBLEED 54 55 #include "shrpx_downstream_connection_pool.h" 56 #include "shrpx_config.h" 57 #include "shrpx_exec.h" 58 59 namespace shrpx { 60 61 class Http2Session; 62 class ConnectBlocker; 63 class AcceptHandler; 64 class Worker; 65 struct WorkerStat; 66 struct TicketKeys; 67 class MemcachedDispatcher; 68 struct UpstreamAddr; 69 70 namespace tls { 71 72 class CertLookupTree; 73 74 } // namespace tls 75 76 struct OCSPUpdateContext { 77 // ocsp response buffer 78 std::vector<uint8_t> resp; 79 // Process running fetch-ocsp-response script 80 Process proc; 81 // index to ConnectionHandler::all_ssl_ctx_, which points to next 82 // SSL_CTX to update ocsp response cache. 83 size_t next; 84 ev_child chldev; 85 ev_io rev; 86 // errno encountered while processing response 87 int error; 88 }; 89 90 // SerialEvent is an event sent from Worker thread. 91 enum class SerialEventType { 92 NONE, 93 REPLACE_DOWNSTREAM, 94 }; 95 96 struct SerialEvent { 97 // ctor for event uses DownstreamConfig SerialEventSerialEvent98 SerialEvent(SerialEventType type, 99 const std::shared_ptr<DownstreamConfig> &downstreamconf) 100 : type(type), downstreamconf(downstreamconf) {} 101 102 SerialEventType type; 103 std::shared_ptr<DownstreamConfig> downstreamconf; 104 }; 105 106 #ifdef ENABLE_HTTP3 107 # ifdef HAVE_LIBBPF 108 struct BPFRef { 109 bpf_object *obj; 110 bpf_map *reuseport_array; 111 bpf_map *worker_id_map; 112 }; 113 # endif // HAVE_LIBBPF 114 115 // QUIC IPC message type. 116 enum class QUICIPCType { 117 NONE, 118 // Send forwarded QUIC UDP datagram and its metadata. 119 DGRAM_FORWARD, 120 }; 121 122 // WorkerProcesses which are in graceful shutdown period. 123 struct QUICLingeringWorkerProcess { QUICLingeringWorkerProcessQUICLingeringWorkerProcess124 QUICLingeringWorkerProcess(std::vector<WorkerID> worker_ids, int quic_ipc_fd) 125 : worker_ids{std::move(worker_ids)}, quic_ipc_fd{quic_ipc_fd} {} 126 127 std::vector<WorkerID> worker_ids; 128 // Socket to send QUIC IPC message to this worker process. 129 int quic_ipc_fd; 130 }; 131 #endif // ENABLE_HTTP3 132 133 class ConnectionHandler { 134 public: 135 ConnectionHandler(struct ev_loop *loop, std::mt19937 &gen); 136 ~ConnectionHandler(); 137 int handle_connection(int fd, sockaddr *addr, int addrlen, 138 const UpstreamAddr *faddr); 139 // Creates Worker object for single threaded configuration. 140 int create_single_worker(); 141 // Creates |num| Worker objects for multi threaded configuration. 142 // The |num| must be strictly more than 1. 143 int create_worker_thread(size_t num); 144 void 145 set_ticket_keys_to_worker(const std::shared_ptr<TicketKeys> &ticket_keys); 146 void worker_reopen_log_files(); 147 void set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys); 148 const std::shared_ptr<TicketKeys> &get_ticket_keys() const; 149 struct ev_loop *get_loop() const; 150 Worker *get_single_worker() const; 151 void add_acceptor(std::unique_ptr<AcceptHandler> h); 152 void delete_acceptor(); 153 void enable_acceptor(); 154 void disable_acceptor(); 155 void sleep_acceptor(ev_tstamp t); 156 void accept_pending_connection(); 157 void graceful_shutdown_worker(); 158 void set_graceful_shutdown(bool f); 159 bool get_graceful_shutdown() const; 160 void join_worker(); 161 162 // Cancels ocsp update process 163 void cancel_ocsp_update(); 164 // Starts ocsp update for certificate |cert_file|. 165 int start_ocsp_update(const char *cert_file); 166 // Reads incoming data from ocsp update process 167 void read_ocsp_chunk(); 168 // Handles the completion of one ocsp update 169 void handle_ocsp_complete(); 170 // Resets ocsp_; 171 void reset_ocsp(); 172 // Proceeds to the next certificate's ocsp update. If all 173 // certificates' ocsp update has been done, schedule next ocsp 174 // update. 175 void proceed_next_cert_ocsp(); 176 177 void set_tls_ticket_key_memcached_dispatcher( 178 std::unique_ptr<MemcachedDispatcher> dispatcher); 179 180 MemcachedDispatcher *get_tls_ticket_key_memcached_dispatcher() const; 181 void on_tls_ticket_key_network_error(ev_timer *w); 182 void on_tls_ticket_key_not_found(ev_timer *w); 183 void 184 on_tls_ticket_key_get_success(const std::shared_ptr<TicketKeys> &ticket_keys, 185 ev_timer *w); 186 void schedule_next_tls_ticket_key_memcached_get(ev_timer *w); 187 SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx(); 188 // Returns the SSL_CTX at all_ssl_ctx_[idx]. This does not perform 189 // array bound checking. 190 SSL_CTX *get_ssl_ctx(size_t idx) const; 191 192 const std::vector<SSL_CTX *> &get_indexed_ssl_ctx(size_t idx) const; 193 #ifdef ENABLE_HTTP3 194 const std::vector<SSL_CTX *> &get_quic_indexed_ssl_ctx(size_t idx) const; 195 196 int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr, 197 const Address &local_addr, const ngtcp2_pkt_info &pi, 198 const WorkerID &wid, std::span<const uint8_t> data); 199 200 void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms); 201 const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const; 202 203 void set_worker_ids(std::vector<WorkerID> worker_ids); 204 Worker *find_worker(const WorkerID &wid) const; 205 206 void set_quic_lingering_worker_processes( 207 const std::vector<QUICLingeringWorkerProcess> &quic_lwps); 208 209 // Return matching QUICLingeringWorkerProcess which has a Worker ID 210 // such that |dcid| starts with it. If no such 211 // QUICLingeringWorkerProcess, it returns nullptr. 212 QUICLingeringWorkerProcess * 213 match_quic_lingering_worker_process_worker_id(const WorkerID &wid); 214 215 int forward_quic_packet_to_lingering_worker_process( 216 QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, 217 const Address &local_addr, const ngtcp2_pkt_info &pi, 218 std::span<const uint8_t> data); 219 220 void set_quic_ipc_fd(int fd); 221 222 int quic_ipc_read(); 223 224 # ifdef HAVE_LIBBPF 225 std::vector<BPFRef> &get_quic_bpf_refs(); 226 void unload_bpf_objects(); 227 # endif // HAVE_LIBBPF 228 #endif // ENABLE_HTTP3 229 230 #ifdef HAVE_NEVERBLEED 231 void set_neverbleed(neverbleed_t *nb); 232 #endif // HAVE_NEVERBLEED 233 234 // Send SerialEvent SerialEventType::REPLACE_DOWNSTREAM to this 235 // object. 236 void send_replace_downstream( 237 const std::shared_ptr<DownstreamConfig> &downstreamconf); 238 // Internal function to send |ev| to this object. 239 void send_serial_event(SerialEvent ev); 240 // Handles SerialEvents received. 241 void handle_serial_event(); 242 // Sends WorkerEvent to make them replace downstream. 243 void 244 worker_replace_downstream(std::shared_ptr<DownstreamConfig> downstreamconf); 245 246 void set_enable_acceptor_on_ocsp_completion(bool f); 247 248 private: 249 // Stores all SSL_CTX objects. 250 std::vector<SSL_CTX *> all_ssl_ctx_; 251 // Stores all SSL_CTX objects in a way that its index is stored in 252 // cert_tree. The SSL_CTXs stored in the same index share the same 253 // hostname, but could have different signature algorithm. The 254 // selection among them are performed by hostname presented by SNI, 255 // and signature algorithm presented by client. 256 std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_; 257 #ifdef ENABLE_HTTP3 258 std::vector<WorkerID> worker_ids_; 259 std::vector<WorkerID> lingering_worker_ids_; 260 int quic_ipc_fd_; 261 std::vector<QUICLingeringWorkerProcess> quic_lingering_worker_processes_; 262 # ifdef HAVE_LIBBPF 263 std::vector<BPFRef> quic_bpf_refs_; 264 # endif // HAVE_LIBBPF 265 std::shared_ptr<QUICKeyingMaterials> quic_keying_materials_; 266 std::vector<SSL_CTX *> quic_all_ssl_ctx_; 267 std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_; 268 #endif // ENABLE_HTTP3 269 OCSPUpdateContext ocsp_; 270 std::mt19937 &gen_; 271 // ev_loop for each worker 272 std::vector<struct ev_loop *> worker_loops_; 273 // Worker instances when multi threaded mode (-nN, N >= 2) is used. 274 // If at least one frontend enables API request, we allocate 1 275 // additional worker dedicated to API request . 276 std::vector<std::unique_ptr<Worker>> workers_; 277 // mutex for serial event resive buffer handling 278 std::mutex serial_event_mu_; 279 // SerialEvent receive buffer 280 std::vector<SerialEvent> serial_events_; 281 // Worker instance used when single threaded mode (-n1) is used. 282 // Otherwise, nullptr and workers_ has instances of Worker instead. 283 std::unique_ptr<Worker> single_worker_; 284 std::unique_ptr<tls::CertLookupTree> cert_tree_; 285 #ifdef ENABLE_HTTP3 286 std::unique_ptr<tls::CertLookupTree> quic_cert_tree_; 287 #endif // ENABLE_HTTP3 288 std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_; 289 // Current TLS session ticket keys. Note that TLS connection does 290 // not refer to this field directly. They use TicketKeys object in 291 // Worker object. 292 std::shared_ptr<TicketKeys> ticket_keys_; 293 struct ev_loop *loop_; 294 std::vector<std::unique_ptr<AcceptHandler>> acceptors_; 295 #ifdef HAVE_NEVERBLEED 296 neverbleed_t *nb_; 297 #endif // HAVE_NEVERBLEED 298 ev_timer disable_acceptor_timer_; 299 ev_timer ocsp_timer_; 300 ev_async thread_join_asyncev_; 301 ev_async serial_event_asyncev_; 302 #ifndef NOTHREADS 303 std::future<void> thread_join_fut_; 304 #endif // NOTHREADS 305 size_t tls_ticket_key_memcached_get_retry_count_; 306 size_t tls_ticket_key_memcached_fail_count_; 307 unsigned int worker_round_robin_cnt_; 308 bool graceful_shutdown_; 309 // true if acceptors should be enabled after the initial ocsp update 310 // has finished. 311 bool enable_acceptor_on_ocsp_completion_; 312 }; 313 314 } // namespace shrpx 315 316 #endif // SHRPX_CONNECTION_HANDLER_H 317