• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2#
3# Copyright 2022 gRPC authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import argparse
18import csv
19import glob
20import math
21import multiprocessing
22import os
23import pathlib
24import re
25import shutil
26import subprocess
27import sys
28
29sys.path.append(
30    os.path.join(
31        os.path.dirname(sys.argv[0]), "..", "..", "run_tests", "python_utils"
32    )
33)
34import check_on_pr
35
36argp = argparse.ArgumentParser(description="Perform diff on memory benchmarks")
37
38argp.add_argument(
39    "-d",
40    "--diff_base",
41    type=str,
42    help="Commit or branch to compare the current one to",
43)
44
45argp.add_argument("-j", "--jobs", type=int, default=multiprocessing.cpu_count())
46
47args = argp.parse_args()
48
49_INTERESTING = {
50    "call/client": (
51        rb"client call memory usage: ([0-9\.]+) bytes per call",
52        float,
53    ),
54    "call/server": (
55        rb"server call memory usage: ([0-9\.]+) bytes per call",
56        float,
57    ),
58    "channel/client": (
59        rb"client channel memory usage: ([0-9\.]+) bytes per channel",
60        float,
61    ),
62    "channel/server": (
63        rb"server channel memory usage: ([0-9\.]+) bytes per channel",
64        float,
65    ),
66    "call/xds_client": (
67        rb"xds client call memory usage: ([0-9\.]+) bytes per call",
68        float,
69    ),
70    "call/xds_server": (
71        rb"xds server call memory usage: ([0-9\.]+) bytes per call",
72        float,
73    ),
74    "channel/xds_client": (
75        rb"xds client channel memory usage: ([0-9\.]+) bytes per channel",
76        float,
77    ),
78    "channel/xds_server": (
79        rb"xds server channel memory usage: ([0-9\.]+) bytes per channel",
80        float,
81    ),
82    "channel_multi_address/xds_client": (
83        rb"xds multi_address client channel memory usage: ([0-9\.]+) bytes per channel",
84        float,
85    ),
86}
87
88_SCENARIOS = {
89    "default": [],
90    "minstack": ["--scenario_config=minstack"],
91    "chaotic_good": ["--scenario_config=chaotic_good"],
92}
93
94_BENCHMARKS = {
95    "call": ["--benchmark_names=call", "--size=50000"],
96    "channel": ["--benchmark_names=channel", "--size=10000"],
97    "channel_multi_address": [
98        "--benchmark_names=channel_multi_address",
99        "--size=10000",
100    ],
101}
102
103
104def _run():
105    """Build with Bazel, then run, and extract interesting lines from the output."""
106    subprocess.check_call(
107        [
108            "tools/bazel",
109            "build",
110            "-c",
111            "opt",
112            "test/core/memory_usage/memory_usage_test",
113        ]
114    )
115    ret = {}
116    for name, benchmark_args in _BENCHMARKS.items():
117        for use_xds in (False, True):
118            for scenario, extra_args in _SCENARIOS.items():
119                # TODO(chenancy) Remove when minstack is implemented for channel
120                if name == "channel" and scenario == "minstack":
121                    continue
122                if name == "channel_multi_address" and not use_xds:
123                    continue
124                argv = (
125                    ["bazel-bin/test/core/memory_usage/memory_usage_test"]
126                    + benchmark_args
127                    + extra_args
128                )
129                if use_xds:
130                    argv.append("--use_xds")
131                try:
132                    output = subprocess.check_output(argv)
133                except subprocess.CalledProcessError as e:
134                    print("Error running benchmark:", e)
135                    continue
136                for line in output.splitlines():
137                    for key, (pattern, conversion) in _INTERESTING.items():
138                        m = re.match(pattern, line)
139                        if m:
140                            ret[scenario + ": " + key] = conversion(m.group(1))
141    return ret
142
143
144cur = _run()
145old = None
146
147if args.diff_base:
148    where_am_i = (
149        subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"])
150        .decode()
151        .strip()
152    )
153    # checkout the diff base (="old")
154    subprocess.check_call(["git", "checkout", args.diff_base])
155    try:
156        old = _run()
157    finally:
158        # restore the original revision (="cur")
159        subprocess.check_call(["git", "checkout", where_am_i])
160
161text = ""
162if old is None:
163    print(cur)
164    for key, value in sorted(cur.items()):
165        text += "{}: {}\n".format(key, value)
166else:
167    print(cur, old)
168    call_diff_size = 0
169    channel_diff_size = 0
170    for scenario in _SCENARIOS.keys():
171        for key, value in sorted(_INTERESTING.items()):
172            key = scenario + ": " + key
173            if key in cur:
174                if key not in old:
175                    text += "{}: {}\n".format(key, cur[key])
176                else:
177                    text += "{}: {} -> {}\n".format(key, old[key], cur[key])
178                    if "call" in key:
179                        call_diff_size += cur[key] - old[key]
180                    else:
181                        channel_diff_size += cur[key] - old[key]
182
183    print("CALL_DIFF_SIZE: %f" % call_diff_size)
184    print("CHANNEL_DIFF_SIZE: %f" % channel_diff_size)
185    check_on_pr.label_increase_decrease_on_pr(
186        "per-call-memory", call_diff_size, 64
187    )
188    check_on_pr.label_increase_decrease_on_pr(
189        "per-channel-memory", channel_diff_size, 1000
190    )
191    # TODO(chennancy)Change significant value when minstack also runs for channel
192
193print(text)
194check_on_pr.check_on_pr("Memory Difference", "```\n%s\n```" % text)
195