1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import inspect 28import logging 29import os 30import subprocess 31from datetime import timedelta, datetime 32from json import JSONEncoder 33import time 34from typing import List, Union, Optional 35 36from .curl import CurlClient, ExecResult 37from .env import Env 38 39 40log = logging.getLogger(__name__) 41 42 43class Httpd: 44 45 MODULES = [ 46 'log_config', 'logio', 'unixd', 'version', 'watchdog', 47 'authn_core', 'authz_user', 'authz_core', 'authz_host', 48 'env', 'filter', 'headers', 'mime', 49 'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect', 50 'mpm_event', 51 ] 52 COMMON_MODULES_DIRS = [ 53 '/usr/lib/apache2/modules', # debian 54 '/usr/libexec/apache2/', # macos 55 ] 56 57 MOD_CURLTEST = None 58 59 def __init__(self, env: Env): 60 self.env = env 61 self._cmd = env.apachectl 62 self._apache_dir = os.path.join(env.gen_dir, 'apache') 63 self._run_dir = os.path.join(self._apache_dir, 'run') 64 self._lock_dir = os.path.join(self._apache_dir, 'locks') 65 self._docs_dir = os.path.join(self._apache_dir, 'docs') 66 self._conf_dir = os.path.join(self._apache_dir, 'conf') 67 self._conf_file = os.path.join(self._conf_dir, 'test.conf') 68 self._logs_dir = os.path.join(self._apache_dir, 'logs') 69 self._error_log = os.path.join(self._logs_dir, 'error_log') 70 self._tmp_dir = os.path.join(self._apache_dir, 'tmp') 71 self._mods_dir = None 72 self._extra_configs = {} 73 assert env.apxs 74 p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'], 75 capture_output=True, text=True) 76 if p.returncode != 0: 77 raise Exception(f'{env.apxs} failed to query libexecdir: {p}') 78 self._mods_dir = p.stdout.strip() 79 if self._mods_dir is None: 80 raise Exception(f'apache modules dir cannot be found') 81 if not os.path.exists(self._mods_dir): 82 raise Exception(f'apache modules dir does not exist: {self._mods_dir}') 83 self._process = None 84 self._rmf(self._error_log) 85 self._init_curltest() 86 87 @property 88 def docs_dir(self): 89 return self._docs_dir 90 91 def clear_logs(self): 92 self._rmf(self._error_log) 93 94 def exists(self): 95 return os.path.exists(self._cmd) 96 97 def set_extra_config(self, domain: str, lines: Optional[Union[str, List[str]]]): 98 if lines is None: 99 self._extra_configs.pop(domain, None) 100 else: 101 self._extra_configs[domain] = lines 102 103 def clear_extra_configs(self): 104 self._extra_configs = {} 105 106 def _run(self, args, intext=''): 107 env = {} 108 for key, val in os.environ.items(): 109 env[key] = val 110 env['APACHE_RUN_DIR'] = self._run_dir 111 env['APACHE_RUN_USER'] = os.environ['USER'] 112 env['APACHE_LOCK_DIR'] = self._lock_dir 113 env['APACHE_CONFDIR'] = self._apache_dir 114 p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, 115 cwd=self.env.gen_dir, 116 input=intext.encode() if intext else None, 117 env=env) 118 start = datetime.now() 119 return ExecResult(args=args, exit_code=p.returncode, 120 stdout=p.stdout.decode().splitlines(), 121 stderr=p.stderr.decode().splitlines(), 122 duration=datetime.now() - start) 123 124 def _apachectl(self, cmd: str): 125 args = [self.env.apachectl, 126 "-d", self._apache_dir, 127 "-f", self._conf_file, 128 "-k", cmd] 129 return self._run(args=args) 130 131 def start(self): 132 if self._process: 133 self.stop() 134 self._write_config() 135 with open(self._error_log, 'a') as fd: 136 fd.write('start of server\n') 137 with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd: 138 fd.write('start of server\n') 139 r = self._apachectl('start') 140 if r.exit_code != 0: 141 log.error(f'failed to start httpd: {r}') 142 return False 143 return self.wait_live(timeout=timedelta(seconds=5)) 144 145 def stop(self): 146 r = self._apachectl('stop') 147 if r.exit_code == 0: 148 return self.wait_dead(timeout=timedelta(seconds=5)) 149 return r.exit_code == 0 150 151 def restart(self): 152 self.stop() 153 return self.start() 154 155 def reload(self): 156 self._write_config() 157 r = self._apachectl("graceful") 158 if r.exit_code != 0: 159 log.error(f'failed to reload httpd: {r}') 160 return self.wait_live(timeout=timedelta(seconds=5)) 161 162 def wait_dead(self, timeout: timedelta): 163 curl = CurlClient(env=self.env, run_dir=self._tmp_dir) 164 try_until = datetime.now() + timeout 165 while datetime.now() < try_until: 166 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 167 if r.exit_code != 0: 168 return True 169 time.sleep(.1) 170 log.debug(f"Server still responding after {timeout}") 171 return False 172 173 def wait_live(self, timeout: timedelta): 174 curl = CurlClient(env=self.env, run_dir=self._tmp_dir) 175 try_until = datetime.now() + timeout 176 while datetime.now() < try_until: 177 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 178 if r.exit_code == 0: 179 return True 180 time.sleep(.1) 181 log.debug(f"Server still not responding after {timeout}") 182 return False 183 184 def _rmf(self, path): 185 if os.path.exists(path): 186 return os.remove(path) 187 188 def _mkpath(self, path): 189 if not os.path.exists(path): 190 return os.makedirs(path) 191 192 def _write_config(self): 193 domain1 = self.env.domain1 194 creds1 = self.env.get_credentials(domain1) 195 domain2 = self.env.domain2 196 creds2 = self.env.get_credentials(domain2) 197 proxy_domain = self.env.proxy_domain 198 proxy_creds = self.env.get_credentials(proxy_domain) 199 self._mkpath(self._conf_dir) 200 self._mkpath(self._logs_dir) 201 self._mkpath(self._tmp_dir) 202 self._mkpath(os.path.join(self._docs_dir, 'two')) 203 with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd: 204 data = { 205 'server': f'{domain1}', 206 } 207 fd.write(JSONEncoder().encode(data)) 208 with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd: 209 data = { 210 'server': f'{domain2}', 211 } 212 fd.write(JSONEncoder().encode(data)) 213 with open(self._conf_file, 'w') as fd: 214 for m in self.MODULES: 215 if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')): 216 fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n') 217 if Httpd.MOD_CURLTEST is not None: 218 fd.write(f'LoadModule curltest_module \"{Httpd.MOD_CURLTEST}\"\n') 219 conf = [ # base server config 220 f'ServerRoot "{self._apache_dir}"', 221 f'DefaultRuntimeDir logs', 222 f'PidFile httpd.pid', 223 f'ErrorLog {self._error_log}', 224 f'LogLevel {self._get_log_level()}', 225 f'LogLevel http:trace4', 226 f'LogLevel proxy:trace4', 227 f'LogLevel proxy_http:trace4', 228 f'H2MinWorkers 16', 229 f'H2MaxWorkers 128', 230 f'H2Direct on', 231 f'Listen {self.env.http_port}', 232 f'Listen {self.env.https_port}', 233 f'Listen {self.env.proxy_port}', 234 f'Listen {self.env.proxys_port}', 235 f'TypesConfig "{self._conf_dir}/mime.types', 236 ] 237 if 'base' in self._extra_configs: 238 conf.extend(self._extra_configs['base']) 239 conf.extend([ # plain http host for domain1 240 f'<VirtualHost *:{self.env.http_port}>', 241 f' ServerName {domain1}', 242 f' ServerAlias localhost', 243 f' DocumentRoot "{self._docs_dir}"', 244 f' Protocols h2c http/1.1', 245 ]) 246 conf.extend(self._curltest_conf()) 247 conf.extend([ 248 f'</VirtualHost>', 249 f'', 250 ]) 251 conf.extend([ # https host for domain1, h1 + h2 252 f'<VirtualHost *:{self.env.https_port}>', 253 f' ServerName {domain1}', 254 f' Protocols h2 http/1.1', 255 f' SSLEngine on', 256 f' SSLCertificateFile {creds1.cert_file}', 257 f' SSLCertificateKeyFile {creds1.pkey_file}', 258 f' DocumentRoot "{self._docs_dir}"', 259 ]) 260 conf.extend(self._curltest_conf()) 261 if domain1 in self._extra_configs: 262 conf.extend(self._extra_configs[domain1]) 263 conf.extend([ 264 f'</VirtualHost>', 265 f'', 266 ]) 267 conf.extend([ # https host for domain2, no h2 268 f'<VirtualHost *:{self.env.https_port}>', 269 f' ServerName {domain2}', 270 f' Protocols http/1.1', 271 f' SSLEngine on', 272 f' SSLCertificateFile {creds2.cert_file}', 273 f' SSLCertificateKeyFile {creds2.pkey_file}', 274 f' DocumentRoot "{self._docs_dir}/two"', 275 ]) 276 conf.extend(self._curltest_conf()) 277 if domain2 in self._extra_configs: 278 conf.extend(self._extra_configs[domain2]) 279 conf.extend([ 280 f'</VirtualHost>', 281 f'', 282 ]) 283 conf.extend([ # http forward proxy 284 f'<VirtualHost *:{self.env.proxy_port}>', 285 f' ServerName {proxy_domain}', 286 f' Protocols h2c, http/1.1', 287 f' ProxyRequests On', 288 f' ProxyVia On', 289 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 290 f' <Proxy "*">', 291 f' Require ip 127.0.0.1', 292 f' </Proxy>', 293 f'</VirtualHost>', 294 ]) 295 conf.extend([ # https forward proxy 296 f'<VirtualHost *:{self.env.proxys_port}>', 297 f' ServerName {proxy_domain}', 298 f' Protocols h2, http/1.1', 299 f' SSLEngine on', 300 f' SSLCertificateFile {proxy_creds.cert_file}', 301 f' SSLCertificateKeyFile {proxy_creds.pkey_file}', 302 f' ProxyRequests On', 303 f' ProxyVia On', 304 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 305 f' <Proxy "*">', 306 f' Require ip 127.0.0.1', 307 f' </Proxy>', 308 f'</VirtualHost>', 309 ]) 310 fd.write("\n".join(conf)) 311 with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd: 312 fd.write("\n".join([ 313 'text/html html', 314 'application/json json', 315 '' 316 ])) 317 318 def _get_log_level(self): 319 #if self.env.verbose > 3: 320 # return 'trace2' 321 #if self.env.verbose > 2: 322 # return 'trace1' 323 #if self.env.verbose > 1: 324 # return 'debug' 325 return 'info' 326 327 def _curltest_conf(self) -> List[str]: 328 if Httpd.MOD_CURLTEST is not None: 329 return [ 330 f' <Location /curltest/echo>', 331 f' SetHandler curltest-echo', 332 f' </Location>', 333 f' <Location /curltest/put>', 334 f' SetHandler curltest-put', 335 f' </Location>', 336 f' <Location /curltest/tweak>', 337 f' SetHandler curltest-tweak', 338 f' </Location>', 339 ] 340 return [] 341 342 def _init_curltest(self): 343 if Httpd.MOD_CURLTEST is not None: 344 return 345 local_dir = os.path.dirname(inspect.getfile(Httpd)) 346 p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'], 347 capture_output=True, 348 cwd=os.path.join(local_dir, 'mod_curltest')) 349 rv = p.returncode 350 if rv != 0: 351 log.error(f"compiling mod_curltest failed: {p.stderr}") 352 raise Exception(f"compiling mod_curltest failed: {p.stderr}") 353 Httpd.MOD_CURLTEST = os.path.join( 354 local_dir, 'mod_curltest/.libs/mod_curltest.so') 355