1 /* 2 * nghttp2 - HTTP/2 C Library 3 * 4 * Copyright (c) 2012 Tatsuhiro Tsujikawa 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining 7 * a copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sublicense, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice shall be 15 * included in all copies or substantial portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 */ 25 #ifndef SHRPX_CONNECTION_HANDLER_H 26 #define SHRPX_CONNECTION_HANDLER_H 27 28 #include "shrpx.h" 29 30 #include <sys/types.h> 31 #ifdef HAVE_SYS_SOCKET_H 32 # include <sys/socket.h> 33 #endif // HAVE_SYS_SOCKET_H 34 35 #include <mutex> 36 #include <memory> 37 #include <vector> 38 #include <random> 39 #ifndef NOTHREADS 40 # include <future> 41 #endif // NOTHREADS 42 43 #ifdef HAVE_LIBBPF 44 # include <bpf/libbpf.h> 45 #endif // HAVE_LIBBPF 46 47 #include <openssl/ssl.h> 48 49 #include <ev.h> 50 51 #ifdef HAVE_NEVERBLEED 52 # include <neverbleed.h> 53 #endif // HAVE_NEVERBLEED 54 55 #include "shrpx_downstream_connection_pool.h" 56 #include "shrpx_config.h" 57 #include "shrpx_exec.h" 58 59 namespace shrpx { 60 61 class Http2Session; 62 class ConnectBlocker; 63 class AcceptHandler; 64 class Worker; 65 struct WorkerStat; 66 struct TicketKeys; 67 class MemcachedDispatcher; 68 struct UpstreamAddr; 69 70 namespace tls { 71 72 class CertLookupTree; 73 74 } // namespace tls 75 76 struct OCSPUpdateContext { 77 // ocsp response buffer 78 std::vector<uint8_t> resp; 79 // Process running fetch-ocsp-response script 80 Process proc; 81 // index to ConnectionHandler::all_ssl_ctx_, which points to next 82 // SSL_CTX to update ocsp response cache. 83 size_t next; 84 ev_child chldev; 85 ev_io rev; 86 // errno encountered while processing response 87 int error; 88 }; 89 90 // SerialEvent is an event sent from Worker thread. 91 enum class SerialEventType { 92 NONE, 93 REPLACE_DOWNSTREAM, 94 }; 95 96 struct SerialEvent { 97 // ctor for event uses DownstreamConfig SerialEventSerialEvent98 SerialEvent(SerialEventType type, 99 const std::shared_ptr<DownstreamConfig> &downstreamconf) 100 : type(type), downstreamconf(downstreamconf) {} 101 102 SerialEventType type; 103 std::shared_ptr<DownstreamConfig> downstreamconf; 104 }; 105 106 #ifdef ENABLE_HTTP3 107 # ifdef HAVE_LIBBPF 108 struct BPFRef { 109 bpf_object *obj; 110 int reuseport_array; 111 int cid_prefix_map; 112 }; 113 # endif // HAVE_LIBBPF 114 115 // QUIC IPC message type. 116 enum class QUICIPCType { 117 NONE, 118 // Send forwarded QUIC UDP datagram and its metadata. 119 DGRAM_FORWARD, 120 }; 121 122 // WorkerProcesses which are in graceful shutdown period. 123 struct QUICLingeringWorkerProcess { QUICLingeringWorkerProcessQUICLingeringWorkerProcess124 QUICLingeringWorkerProcess( 125 std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes, 126 int quic_ipc_fd) 127 : cid_prefixes{std::move(cid_prefixes)}, quic_ipc_fd{quic_ipc_fd} {} 128 129 std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes; 130 // Socket to send QUIC IPC message to this worker process. 131 int quic_ipc_fd; 132 }; 133 #endif // ENABLE_HTTP3 134 135 class ConnectionHandler { 136 public: 137 ConnectionHandler(struct ev_loop *loop, std::mt19937 &gen); 138 ~ConnectionHandler(); 139 int handle_connection(int fd, sockaddr *addr, int addrlen, 140 const UpstreamAddr *faddr); 141 // Creates Worker object for single threaded configuration. 142 int create_single_worker(); 143 // Creates |num| Worker objects for multi threaded configuration. 144 // The |num| must be strictly more than 1. 145 int create_worker_thread(size_t num); 146 void 147 set_ticket_keys_to_worker(const std::shared_ptr<TicketKeys> &ticket_keys); 148 void worker_reopen_log_files(); 149 void set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys); 150 const std::shared_ptr<TicketKeys> &get_ticket_keys() const; 151 struct ev_loop *get_loop() const; 152 Worker *get_single_worker() const; 153 void add_acceptor(std::unique_ptr<AcceptHandler> h); 154 void delete_acceptor(); 155 void enable_acceptor(); 156 void disable_acceptor(); 157 void sleep_acceptor(ev_tstamp t); 158 void accept_pending_connection(); 159 void graceful_shutdown_worker(); 160 void set_graceful_shutdown(bool f); 161 bool get_graceful_shutdown() const; 162 void join_worker(); 163 164 // Cancels ocsp update process 165 void cancel_ocsp_update(); 166 // Starts ocsp update for certificate |cert_file|. 167 int start_ocsp_update(const char *cert_file); 168 // Reads incoming data from ocsp update process 169 void read_ocsp_chunk(); 170 // Handles the completion of one ocsp update 171 void handle_ocsp_complete(); 172 // Resets ocsp_; 173 void reset_ocsp(); 174 // Proceeds to the next certificate's ocsp update. If all 175 // certificates' ocsp update has been done, schedule next ocsp 176 // update. 177 void proceed_next_cert_ocsp(); 178 179 void set_tls_ticket_key_memcached_dispatcher( 180 std::unique_ptr<MemcachedDispatcher> dispatcher); 181 182 MemcachedDispatcher *get_tls_ticket_key_memcached_dispatcher() const; 183 void on_tls_ticket_key_network_error(ev_timer *w); 184 void on_tls_ticket_key_not_found(ev_timer *w); 185 void 186 on_tls_ticket_key_get_success(const std::shared_ptr<TicketKeys> &ticket_keys, 187 ev_timer *w); 188 void schedule_next_tls_ticket_key_memcached_get(ev_timer *w); 189 SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx(); 190 // Returns the SSL_CTX at all_ssl_ctx_[idx]. This does not perform 191 // array bound checking. 192 SSL_CTX *get_ssl_ctx(size_t idx) const; 193 194 const std::vector<SSL_CTX *> &get_indexed_ssl_ctx(size_t idx) const; 195 #ifdef ENABLE_HTTP3 196 const std::vector<SSL_CTX *> &get_quic_indexed_ssl_ctx(size_t idx) const; 197 198 int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr, 199 const Address &local_addr, const ngtcp2_pkt_info &pi, 200 const uint8_t *cid_prefix, const uint8_t *data, 201 size_t datalen); 202 203 void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms); 204 const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const; 205 206 void set_cid_prefixes( 207 const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> 208 &cid_prefixes); 209 210 void set_quic_lingering_worker_processes( 211 const std::vector<QUICLingeringWorkerProcess> &quic_lwps); 212 213 // Return matching QUICLingeringWorkerProcess which has a CID prefix 214 // such that |dcid| starts with it. If no such 215 // QUICLingeringWorkerProcess, it returns nullptr. 216 QUICLingeringWorkerProcess * 217 match_quic_lingering_worker_process_cid_prefix(const uint8_t *dcid, 218 size_t dcidlen); 219 220 int forward_quic_packet_to_lingering_worker_process( 221 QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, 222 const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data, 223 size_t datalen); 224 225 void set_quic_ipc_fd(int fd); 226 227 int quic_ipc_read(); 228 229 # ifdef HAVE_LIBBPF 230 std::vector<BPFRef> &get_quic_bpf_refs(); 231 void unload_bpf_objects(); 232 # endif // HAVE_LIBBPF 233 #endif // ENABLE_HTTP3 234 235 #ifdef HAVE_NEVERBLEED 236 void set_neverbleed(neverbleed_t *nb); 237 #endif // HAVE_NEVERBLEED 238 239 // Send SerialEvent SerialEventType::REPLACE_DOWNSTREAM to this 240 // object. 241 void send_replace_downstream( 242 const std::shared_ptr<DownstreamConfig> &downstreamconf); 243 // Internal function to send |ev| to this object. 244 void send_serial_event(SerialEvent ev); 245 // Handles SerialEvents received. 246 void handle_serial_event(); 247 // Sends WorkerEvent to make them replace downstream. 248 void 249 worker_replace_downstream(std::shared_ptr<DownstreamConfig> downstreamconf); 250 251 void set_enable_acceptor_on_ocsp_completion(bool f); 252 253 private: 254 // Stores all SSL_CTX objects. 255 std::vector<SSL_CTX *> all_ssl_ctx_; 256 // Stores all SSL_CTX objects in a way that its index is stored in 257 // cert_tree. The SSL_CTXs stored in the same index share the same 258 // hostname, but could have different signature algorithm. The 259 // selection among them are performed by hostname presented by SNI, 260 // and signature algorithm presented by client. 261 std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_; 262 #ifdef ENABLE_HTTP3 263 std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes_; 264 std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> 265 lingering_cid_prefixes_; 266 int quic_ipc_fd_; 267 std::vector<QUICLingeringWorkerProcess> quic_lingering_worker_processes_; 268 # ifdef HAVE_LIBBPF 269 std::vector<BPFRef> quic_bpf_refs_; 270 # endif // HAVE_LIBBPF 271 std::shared_ptr<QUICKeyingMaterials> quic_keying_materials_; 272 std::vector<SSL_CTX *> quic_all_ssl_ctx_; 273 std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_; 274 #endif // ENABLE_HTTP3 275 OCSPUpdateContext ocsp_; 276 std::mt19937 &gen_; 277 // ev_loop for each worker 278 std::vector<struct ev_loop *> worker_loops_; 279 // Worker instances when multi threaded mode (-nN, N >= 2) is used. 280 // If at least one frontend enables API request, we allocate 1 281 // additional worker dedicated to API request . 282 std::vector<std::unique_ptr<Worker>> workers_; 283 // mutex for serial event resive buffer handling 284 std::mutex serial_event_mu_; 285 // SerialEvent receive buffer 286 std::vector<SerialEvent> serial_events_; 287 // Worker instance used when single threaded mode (-n1) is used. 288 // Otherwise, nullptr and workers_ has instances of Worker instead. 289 std::unique_ptr<Worker> single_worker_; 290 std::unique_ptr<tls::CertLookupTree> cert_tree_; 291 #ifdef ENABLE_HTTP3 292 std::unique_ptr<tls::CertLookupTree> quic_cert_tree_; 293 #endif // ENABLE_HTTP3 294 std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_; 295 // Current TLS session ticket keys. Note that TLS connection does 296 // not refer to this field directly. They use TicketKeys object in 297 // Worker object. 298 std::shared_ptr<TicketKeys> ticket_keys_; 299 struct ev_loop *loop_; 300 std::vector<std::unique_ptr<AcceptHandler>> acceptors_; 301 #ifdef HAVE_NEVERBLEED 302 neverbleed_t *nb_; 303 #endif // HAVE_NEVERBLEED 304 ev_timer disable_acceptor_timer_; 305 ev_timer ocsp_timer_; 306 ev_async thread_join_asyncev_; 307 ev_async serial_event_asyncev_; 308 #ifndef NOTHREADS 309 std::future<void> thread_join_fut_; 310 #endif // NOTHREADS 311 size_t tls_ticket_key_memcached_get_retry_count_; 312 size_t tls_ticket_key_memcached_fail_count_; 313 unsigned int worker_round_robin_cnt_; 314 bool graceful_shutdown_; 315 // true if acceptors should be enabled after the initial ocsp update 316 // has finished. 317 bool enable_acceptor_on_ocsp_completion_; 318 }; 319 320 } // namespace shrpx 321 322 #endif // SHRPX_CONNECTION_HANDLER_H 323