1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import json 28import logging 29import os 30import psutil 31import re 32import shutil 33import subprocess 34from statistics import mean, fmean 35from datetime import timedelta, datetime 36from typing import List, Optional, Dict, Union, Any 37from urllib.parse import urlparse 38 39from .env import Env 40 41 42log = logging.getLogger(__name__) 43 44 45class RunProfile: 46 47 STAT_KEYS = ['cpu', 'rss', 'vsz'] 48 49 @classmethod 50 def AverageStats(cls, profiles: List['RunProfile']): 51 avg = {} 52 stats = [p.stats for p in profiles] 53 for key in cls.STAT_KEYS: 54 avg[key] = mean([s[key] for s in stats]) 55 return avg 56 57 def __init__(self, pid: int, started_at: datetime, run_dir): 58 self._pid = pid 59 self._started_at = started_at 60 self._duration = timedelta(seconds=0) 61 self._run_dir = run_dir 62 self._samples = [] 63 self._psu = None 64 self._stats = None 65 66 @property 67 def duration(self) -> timedelta: 68 return self._duration 69 70 @property 71 def stats(self) -> Optional[Dict[str,Any]]: 72 return self._stats 73 74 def sample(self): 75 elapsed = datetime.now() - self._started_at 76 try: 77 if self._psu is None: 78 self._psu = psutil.Process(pid=self._pid) 79 mem = self._psu.memory_info() 80 self._samples.append({ 81 'time': elapsed, 82 'cpu': self._psu.cpu_percent(), 83 'vsz': mem.vms, 84 'rss': mem.rss, 85 }) 86 except psutil.NoSuchProcess: 87 pass 88 89 def finish(self): 90 self._duration = datetime.now() - self._started_at 91 if len(self._samples) > 0: 92 weights = [s['time'].total_seconds() for s in self._samples] 93 self._stats = {} 94 for key in self.STAT_KEYS: 95 self._stats[key] = fmean([s[key] for s in self._samples], weights) 96 else: 97 self._stats = None 98 self._psu = None 99 100 def __repr__(self): 101 return f'RunProfile[pid={self._pid}, '\ 102 f'duration={self.duration.total_seconds():.3f}s, '\ 103 f'stats={self.stats}]' 104 105 106class ExecResult: 107 108 def __init__(self, args: List[str], exit_code: int, 109 stdout: List[str], stderr: List[str], 110 duration: Optional[timedelta] = None, 111 with_stats: bool = False, 112 exception: Optional[str] = None, 113 profile: Optional[RunProfile] = None): 114 self._args = args 115 self._exit_code = exit_code 116 self._exception = exception 117 self._stdout = stdout 118 self._stderr = stderr 119 self._profile = profile 120 self._duration = duration if duration is not None else timedelta() 121 self._response = None 122 self._responses = [] 123 self._results = {} 124 self._assets = [] 125 self._stats = [] 126 self._json_out = None 127 self._with_stats = with_stats 128 if with_stats: 129 self._parse_stats() 130 else: 131 # noinspection PyBroadException 132 try: 133 out = ''.join(self._stdout) 134 self._json_out = json.loads(out) 135 except: 136 pass 137 138 def __repr__(self): 139 return f"ExecResult[code={self.exit_code}, exception={self._exception}, "\ 140 f"args={self._args}, stdout={self._stdout}, stderr={self._stderr}]" 141 142 def _parse_stats(self): 143 self._stats = [] 144 for l in self._stdout: 145 try: 146 self._stats.append(json.loads(l)) 147 except: 148 log.error(f'not a JSON stat: {l}') 149 break 150 151 @property 152 def exit_code(self) -> int: 153 return self._exit_code 154 155 @property 156 def args(self) -> List[str]: 157 return self._args 158 159 @property 160 def outraw(self) -> bytes: 161 return ''.join(self._stdout).encode() 162 163 @property 164 def stdout(self) -> str: 165 return ''.join(self._stdout) 166 167 @property 168 def json(self) -> Optional[Dict]: 169 """Output as JSON dictionary or None if not parseable.""" 170 return self._json_out 171 172 @property 173 def stderr(self) -> str: 174 return ''.join(self._stderr) 175 176 @property 177 def trace_lines(self) -> List[str]: 178 return self._stderr 179 180 @property 181 def duration(self) -> timedelta: 182 return self._duration 183 184 @property 185 def profile(self) -> Optional[RunProfile]: 186 return self._profile 187 188 @property 189 def response(self) -> Optional[Dict]: 190 return self._response 191 192 @property 193 def responses(self) -> List[Dict]: 194 return self._responses 195 196 @property 197 def results(self) -> Dict: 198 return self._results 199 200 @property 201 def assets(self) -> List: 202 return self._assets 203 204 @property 205 def with_stats(self) -> bool: 206 return self._with_stats 207 208 @property 209 def stats(self) -> List: 210 return self._stats 211 212 @property 213 def total_connects(self) -> Optional[int]: 214 if len(self.stats): 215 n = 0 216 for stat in self.stats: 217 n += stat['num_connects'] 218 return n 219 return None 220 221 def add_response(self, resp: Dict): 222 self._response = resp 223 self._responses.append(resp) 224 225 def add_results(self, results: Dict): 226 self._results.update(results) 227 if 'response' in results: 228 self.add_response(results['response']) 229 230 def add_assets(self, assets: List): 231 self._assets.extend(assets) 232 233 def check_exit_code(self, code: Union[int, bool]): 234 if code is True: 235 assert self.exit_code == 0, f'expected exit code {code}, '\ 236 f'got {self.exit_code}\n{self.dump_logs()}' 237 elif code is False: 238 assert self.exit_code != 0, f'expected exit code {code}, '\ 239 f'got {self.exit_code}\n{self.dump_logs()}' 240 else: 241 assert self.exit_code == code, f'expected exit code {code}, '\ 242 f'got {self.exit_code}\n{self.dump_logs()}' 243 244 def check_response(self, http_status: Optional[int] = 200, 245 count: Optional[int] = 1, 246 protocol: Optional[str] = None, 247 exitcode: Optional[int] = 0, 248 connect_count: Optional[int] = None): 249 if exitcode: 250 self.check_exit_code(exitcode) 251 if self.with_stats and isinstance(exitcode, int): 252 for idx, x in enumerate(self.stats): 253 if 'exitcode' in x: 254 assert int(x['exitcode']) == exitcode, \ 255 f'response #{idx} exitcode: expected {exitcode}, '\ 256 f'got {x["exitcode"]}\n{self.dump_logs()}' 257 258 if self.with_stats: 259 assert len(self.stats) == count, \ 260 f'response count: expected {count}, ' \ 261 f'got {len(self.stats)}\n{self.dump_logs()}' 262 else: 263 assert len(self.responses) == count, \ 264 f'response count: expected {count}, ' \ 265 f'got {len(self.responses)}\n{self.dump_logs()}' 266 if http_status is not None: 267 if self.with_stats: 268 for idx, x in enumerate(self.stats): 269 assert 'http_code' in x, \ 270 f'response #{idx} reports no http_code\n{self.dump_stat(x)}' 271 assert x['http_code'] == http_status, \ 272 f'response #{idx} http_code: expected {http_status}, '\ 273 f'got {x["http_code"]}\n{self.dump_stat(x)}' 274 else: 275 for idx, x in enumerate(self.responses): 276 assert x['status'] == http_status, \ 277 f'response #{idx} status: expected {http_status},'\ 278 f'got {x["status"]}\n{self.dump_stat(x)}' 279 if protocol is not None: 280 if self.with_stats: 281 http_version = None 282 if protocol == 'HTTP/1.1': 283 http_version = '1.1' 284 elif protocol == 'HTTP/2': 285 http_version = '2' 286 elif protocol == 'HTTP/3': 287 http_version = '3' 288 if http_version is not None: 289 for idx, x in enumerate(self.stats): 290 assert x['http_version'] == http_version, \ 291 f'response #{idx} protocol: expected http/{http_version},' \ 292 f'got version {x["http_version"]}\n{self.dump_stat(x)}' 293 else: 294 for idx, x in enumerate(self.responses): 295 assert x['protocol'] == protocol, \ 296 f'response #{idx} protocol: expected {protocol},'\ 297 f'got {x["protocol"]}\n{self.dump_logs()}' 298 if connect_count is not None: 299 assert self.total_connects == connect_count, \ 300 f'expected {connect_count}, but {self.total_connects} '\ 301 f'were made\n{self.dump_logs()}' 302 303 def check_stats(self, count: int, http_status: Optional[int] = None, 304 exitcode: Optional[int] = None): 305 if exitcode is None: 306 self.check_exit_code(0) 307 assert len(self.stats) == count, \ 308 f'stats count: expected {count}, got {len(self.stats)}\n{self.dump_logs()}' 309 if http_status is not None: 310 for idx, x in enumerate(self.stats): 311 assert 'http_code' in x, \ 312 f'status #{idx} reports no http_code\n{self.dump_stat(x)}' 313 assert x['http_code'] == http_status, \ 314 f'status #{idx} http_code: expected {http_status}, '\ 315 f'got {x["http_code"]}\n{self.dump_stat(x)}' 316 if exitcode is not None: 317 for idx, x in enumerate(self.stats): 318 if 'exitcode' in x: 319 assert x['exitcode'] == exitcode, \ 320 f'status #{idx} exitcode: expected {exitcode}, '\ 321 f'got {x["exitcode"]}\n{self.dump_stat(x)}' 322 323 def dump_logs(self): 324 lines = ['>>--stdout ----------------------------------------------\n'] 325 lines.extend(self._stdout) 326 lines.append('>>--stderr ----------------------------------------------\n') 327 lines.extend(self._stderr) 328 lines.append('<<-------------------------------------------------------\n') 329 return ''.join(lines) 330 331 def dump_stat(self, x): 332 lines = [ 333 'json stat from curl:', 334 json.JSONEncoder(indent=2).encode(x), 335 ] 336 if 'xfer_id' in x: 337 xfer_id = x['xfer_id'] 338 lines.append(f'>>--xfer {xfer_id} trace:\n') 339 lines.extend(self.xfer_trace_for(xfer_id)) 340 else: 341 lines.append('>>--full trace-------------------------------------------\n') 342 lines.extend(self._stderr) 343 lines.append('<<-------------------------------------------------------\n') 344 return ''.join(lines) 345 346 def xfer_trace_for(self, xfer_id) -> List[str]: 347 pat = re.compile(f'^[^[]* \\[{xfer_id}-.*$') 348 return [line for line in self._stderr if pat.match(line)] 349 350 351class CurlClient: 352 353 ALPN_ARG = { 354 'http/0.9': '--http0.9', 355 'http/1.0': '--http1.0', 356 'http/1.1': '--http1.1', 357 'h2': '--http2', 358 'h2c': '--http2', 359 'h3': '--http3-only', 360 } 361 362 def __init__(self, env: Env, run_dir: Optional[str] = None, 363 timeout: Optional[float] = None, silent: bool = False): 364 self.env = env 365 self._timeout = timeout if timeout else env.test_timeout 366 self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl 367 self._run_dir = run_dir if run_dir else os.path.join(env.gen_dir, 'curl') 368 self._stdoutfile = f'{self._run_dir}/curl.stdout' 369 self._stderrfile = f'{self._run_dir}/curl.stderr' 370 self._headerfile = f'{self._run_dir}/curl.headers' 371 self._log_path = f'{self._run_dir}/curl.log' 372 self._silent = silent 373 self._rmrf(self._run_dir) 374 self._mkpath(self._run_dir) 375 376 @property 377 def run_dir(self) -> str: 378 return self._run_dir 379 380 def download_file(self, i: int) -> str: 381 return os.path.join(self.run_dir, f'download_{i}.data') 382 383 def _rmf(self, path): 384 if os.path.exists(path): 385 return os.remove(path) 386 387 def _rmrf(self, path): 388 if os.path.exists(path): 389 return shutil.rmtree(path) 390 391 def _mkpath(self, path): 392 if not os.path.exists(path): 393 return os.makedirs(path) 394 395 def get_proxy_args(self, proto: str = 'http/1.1', 396 proxys: bool = True, tunnel: bool = False, 397 use_ip: bool = False): 398 proxy_name = '127.0.0.1' if use_ip else self.env.proxy_domain 399 if proxys: 400 pport = self.env.pts_port(proto) if tunnel else self.env.proxys_port 401 xargs = [ 402 '--proxy', f'https://{proxy_name}:{pport}/', 403 '--resolve', f'{proxy_name}:{pport}:127.0.0.1', 404 '--proxy-cacert', self.env.ca.cert_file, 405 ] 406 if proto == 'h2': 407 xargs.append('--proxy-http2') 408 else: 409 xargs = [ 410 '--proxy', f'http://{proxy_name}:{self.env.proxy_port}/', 411 '--resolve', f'{proxy_name}:{self.env.proxy_port}:127.0.0.1', 412 ] 413 if tunnel: 414 xargs.append('--proxytunnel') 415 return xargs 416 417 def http_get(self, url: str, extra_args: Optional[List[str]] = None, 418 alpn_proto: Optional[str] = None, 419 def_tracing: bool = True, 420 with_stats: bool = False, 421 with_profile: bool = False): 422 return self._raw(url, options=extra_args, 423 with_stats=with_stats, 424 alpn_proto=alpn_proto, 425 def_tracing=def_tracing, 426 with_profile=with_profile) 427 428 def http_download(self, urls: List[str], 429 alpn_proto: Optional[str] = None, 430 with_stats: bool = True, 431 with_headers: bool = False, 432 with_profile: bool = False, 433 no_save: bool = False, 434 extra_args: List[str] = None): 435 if extra_args is None: 436 extra_args = [] 437 if no_save: 438 extra_args.extend([ 439 '-o', '/dev/null', 440 ]) 441 else: 442 extra_args.extend([ 443 '-o', 'download_#1.data', 444 ]) 445 # remove any existing ones 446 for i in range(100): 447 self._rmf(self.download_file(i)) 448 if with_stats: 449 extra_args.extend([ 450 '-w', '%{json}\\n' 451 ]) 452 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 453 with_stats=with_stats, 454 with_headers=with_headers, 455 with_profile=with_profile) 456 457 def http_upload(self, urls: List[str], data: str, 458 alpn_proto: Optional[str] = None, 459 with_stats: bool = True, 460 with_headers: bool = False, 461 with_profile: bool = False, 462 extra_args: Optional[List[str]] = None): 463 if extra_args is None: 464 extra_args = [] 465 extra_args.extend([ 466 '--data-binary', data, '-o', 'download_#1.data', 467 ]) 468 if with_stats: 469 extra_args.extend([ 470 '-w', '%{json}\\n' 471 ]) 472 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 473 with_stats=with_stats, 474 with_headers=with_headers, 475 with_profile=with_profile) 476 477 def http_delete(self, urls: List[str], 478 alpn_proto: Optional[str] = None, 479 with_stats: bool = True, 480 with_profile: bool = False, 481 extra_args: Optional[List[str]] = None): 482 if extra_args is None: 483 extra_args = [] 484 extra_args.extend([ 485 '-X', 'DELETE', '-o', '/dev/null', 486 ]) 487 if with_stats: 488 extra_args.extend([ 489 '-w', '%{json}\\n' 490 ]) 491 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 492 with_stats=with_stats, 493 with_headers=False, 494 with_profile=with_profile) 495 496 def http_put(self, urls: List[str], data=None, fdata=None, 497 alpn_proto: Optional[str] = None, 498 with_stats: bool = True, 499 with_headers: bool = False, 500 with_profile: bool = False, 501 extra_args: Optional[List[str]] = None): 502 if extra_args is None: 503 extra_args = [] 504 if fdata is not None: 505 extra_args.extend(['-T', fdata]) 506 elif data is not None: 507 extra_args.extend(['-T', '-']) 508 extra_args.extend([ 509 '-o', 'download_#1.data', 510 ]) 511 if with_stats: 512 extra_args.extend([ 513 '-w', '%{json}\\n' 514 ]) 515 return self._raw(urls, intext=data, 516 alpn_proto=alpn_proto, options=extra_args, 517 with_stats=with_stats, 518 with_headers=with_headers, 519 with_profile=with_profile) 520 521 def http_form(self, urls: List[str], form: Dict[str, str], 522 alpn_proto: Optional[str] = None, 523 with_stats: bool = True, 524 with_headers: bool = False, 525 extra_args: Optional[List[str]] = None): 526 if extra_args is None: 527 extra_args = [] 528 for key, val in form.items(): 529 extra_args.extend(['-F', f'{key}={val}']) 530 extra_args.extend([ 531 '-o', 'download_#1.data', 532 ]) 533 if with_stats: 534 extra_args.extend([ 535 '-w', '%{json}\\n' 536 ]) 537 return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, 538 with_stats=with_stats, 539 with_headers=with_headers) 540 541 def ftp_get(self, urls: List[str], 542 with_stats: bool = True, 543 with_profile: bool = False, 544 no_save: bool = False, 545 extra_args: List[str] = None): 546 if extra_args is None: 547 extra_args = [] 548 if no_save: 549 extra_args.extend([ 550 '-o', '/dev/null', 551 ]) 552 else: 553 extra_args.extend([ 554 '-o', 'download_#1.data', 555 ]) 556 # remove any existing ones 557 for i in range(100): 558 self._rmf(self.download_file(i)) 559 if with_stats: 560 extra_args.extend([ 561 '-w', '%{json}\\n' 562 ]) 563 return self._raw(urls, options=extra_args, 564 with_stats=with_stats, 565 with_headers=False, 566 with_profile=with_profile) 567 568 def ftp_ssl_get(self, urls: List[str], 569 with_stats: bool = True, 570 with_profile: bool = False, 571 no_save: bool = False, 572 extra_args: List[str] = None): 573 if extra_args is None: 574 extra_args = [] 575 extra_args.extend([ 576 '--ssl-reqd', 577 ]) 578 return self.ftp_get(urls=urls, with_stats=with_stats, 579 with_profile=with_profile, no_save=no_save, 580 extra_args=extra_args) 581 582 def response_file(self, idx: int): 583 return os.path.join(self._run_dir, f'download_{idx}.data') 584 585 def run_direct(self, args, with_stats: bool = False, with_profile: bool = False): 586 my_args = [self._curl] 587 if with_stats: 588 my_args.extend([ 589 '-w', '%{json}\\n' 590 ]) 591 my_args.extend([ 592 '-o', 'download.data', 593 ]) 594 my_args.extend(args) 595 return self._run(args=my_args, with_stats=with_stats, with_profile=with_profile) 596 597 def _run(self, args, intext='', with_stats: bool = False, with_profile: bool = True): 598 self._rmf(self._stdoutfile) 599 self._rmf(self._stderrfile) 600 self._rmf(self._headerfile) 601 started_at = datetime.now() 602 exception = None 603 profile = None 604 started_at = datetime.now() 605 try: 606 with open(self._stdoutfile, 'w') as cout: 607 with open(self._stderrfile, 'w') as cerr: 608 if with_profile: 609 end_at = started_at + timedelta(seconds=self._timeout) \ 610 if self._timeout else None 611 log.info(f'starting: {args}') 612 p = subprocess.Popen(args, stderr=cerr, stdout=cout, 613 cwd=self._run_dir, shell=False) 614 profile = RunProfile(p.pid, started_at, self._run_dir) 615 if intext is not None and False: 616 p.communicate(input=intext.encode(), timeout=1) 617 ptimeout = 0.0 618 while True: 619 try: 620 p.wait(timeout=ptimeout) 621 break 622 except subprocess.TimeoutExpired: 623 if end_at and datetime.now() >= end_at: 624 p.kill() 625 raise subprocess.TimeoutExpired(cmd=args, timeout=self._timeout) 626 profile.sample() 627 ptimeout = 0.01 628 exitcode = p.returncode 629 profile.finish() 630 log.info(f'done: exit={exitcode}, profile={profile}') 631 else: 632 p = subprocess.run(args, stderr=cerr, stdout=cout, 633 cwd=self._run_dir, shell=False, 634 input=intext.encode() if intext else None, 635 timeout=self._timeout) 636 exitcode = p.returncode 637 except subprocess.TimeoutExpired: 638 now = datetime.now() 639 duration = now - started_at 640 log.warning(f'Timeout at {now} after {duration.total_seconds()}s ' 641 f'(configured {self._timeout}s): {args}') 642 exitcode = -1 643 exception = 'TimeoutExpired' 644 coutput = open(self._stdoutfile).readlines() 645 cerrput = open(self._stderrfile).readlines() 646 return ExecResult(args=args, exit_code=exitcode, exception=exception, 647 stdout=coutput, stderr=cerrput, 648 duration=datetime.now() - started_at, 649 with_stats=with_stats, 650 profile=profile) 651 652 def _raw(self, urls, intext='', timeout=None, options=None, insecure=False, 653 alpn_proto: Optional[str] = None, 654 force_resolve=True, 655 with_stats=False, 656 with_headers=True, 657 def_tracing=True, 658 with_profile=False): 659 args = self._complete_args( 660 urls=urls, timeout=timeout, options=options, insecure=insecure, 661 alpn_proto=alpn_proto, force_resolve=force_resolve, 662 with_headers=with_headers, def_tracing=def_tracing) 663 r = self._run(args, intext=intext, with_stats=with_stats, 664 with_profile=with_profile) 665 if r.exit_code == 0 and with_headers: 666 self._parse_headerfile(self._headerfile, r=r) 667 if r.json: 668 r.response["json"] = r.json 669 return r 670 671 def _complete_args(self, urls, timeout=None, options=None, 672 insecure=False, force_resolve=True, 673 alpn_proto: Optional[str] = None, 674 with_headers: bool = True, 675 def_tracing: bool = True): 676 if not isinstance(urls, list): 677 urls = [urls] 678 679 args = [self._curl, "-s", "--path-as-is"] 680 if with_headers: 681 args.extend(["-D", self._headerfile]) 682 if def_tracing is not False and not self._silent: 683 args.extend(['-v', '--trace-ids', '--trace-time']) 684 if self.env.verbose > 1: 685 args.extend(['--trace-config', 'http/2,http/3,h2-proxy,h1-proxy']) 686 pass 687 688 active_options = options 689 if options is not None and '--next' in options: 690 active_options = options[options.index('--next') + 1:] 691 692 for url in urls: 693 u = urlparse(urls[0]) 694 if options: 695 args.extend(options) 696 if alpn_proto is not None: 697 if alpn_proto not in self.ALPN_ARG: 698 raise Exception(f'unknown ALPN protocol: "{alpn_proto}"') 699 args.append(self.ALPN_ARG[alpn_proto]) 700 701 if u.scheme == 'http': 702 pass 703 elif insecure: 704 args.append('--insecure') 705 elif active_options and "--cacert" in active_options: 706 pass 707 elif u.hostname: 708 args.extend(["--cacert", self.env.ca.cert_file]) 709 710 if force_resolve and u.hostname and u.hostname != 'localhost' \ 711 and not re.match(r'^(\d+|\[|:).*', u.hostname): 712 port = u.port if u.port else 443 713 args.extend(["--resolve", f"{u.hostname}:{port}:127.0.0.1"]) 714 if timeout is not None and int(timeout) > 0: 715 args.extend(["--connect-timeout", str(int(timeout))]) 716 args.append(url) 717 return args 718 719 def _parse_headerfile(self, headerfile: str, r: ExecResult = None) -> ExecResult: 720 lines = open(headerfile).readlines() 721 if r is None: 722 r = ExecResult(args=[], exit_code=0, stdout=[], stderr=[]) 723 724 response = None 725 726 def fin_response(resp): 727 if resp: 728 r.add_response(resp) 729 730 expected = ['status'] 731 for line in lines: 732 line = line.strip() 733 if re.match(r'^$', line): 734 if 'trailer' in expected: 735 # end of trailers 736 fin_response(response) 737 response = None 738 expected = ['status'] 739 elif 'header' in expected: 740 # end of header, another status or trailers might follow 741 expected = ['status', 'trailer'] 742 else: 743 assert False, f"unexpected line: '{line}'" 744 continue 745 if 'status' in expected: 746 # log.debug("reading 1st response line: %s", line) 747 m = re.match(r'^(\S+) (\d+)( .*)?$', line) 748 if m: 749 fin_response(response) 750 response = { 751 "protocol": m.group(1), 752 "status": int(m.group(2)), 753 "description": m.group(3), 754 "header": {}, 755 "trailer": {}, 756 "body": r.outraw 757 } 758 expected = ['header'] 759 continue 760 if 'trailer' in expected: 761 m = re.match(r'^([^:]+):\s*(.*)$', line) 762 if m: 763 response['trailer'][m.group(1).lower()] = m.group(2) 764 continue 765 if 'header' in expected: 766 m = re.match(r'^([^:]+):\s*(.*)$', line) 767 if m: 768 response['header'][m.group(1).lower()] = m.group(2) 769 continue 770 assert False, f"unexpected line: '{line}, expected: {expected}'" 771 772 fin_response(response) 773 return r 774