{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Run Tests on ChromeOS using `test_that`" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This notebook shows how to run tests on a Chromebook using `test_that`. At the end of the test results are collected as a dictionary as well as trace events if you specify so in the target configuration.\n", "\n", "**NOTE**: if you want to receive a token from the benchmark before starting trace collection and power measurement, you need to modify the benchmark such that a specific UDP packet is sent to the host machine. As a reference, consider the following python script and convert it to the language in which the benchmark is written accordingly:\n", "\n", "```python\n", "import socket\n", "sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", "sock.sendto(\"POWER\", (, 1234))\n", "sock.close()\n", "```\n", "\n", "You should put this right before the instruction that starts the execution of the actual workload in the benchmark.\n", "\n", "\n", "In case of Acquarium, you should put this in\n", "\n", "```shell\n", "/trunk/src/third_party/autotest/files/client/site_tests/graphics_WebGLAquarium/graphics_WebGLAcquarium.py:run_fish_test()\n", "```\n", "\n", "Being `CROS_PATH` the ChromeOS chroot." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import logging\n", "from conf import LisaLogging\n", "LisaLogging.setup()" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import getpass\n", "from subprocess import Popen, PIPE\n", "\n", "import os\n", "import pandas as pd\n", "import scipy.integrate as integrate\n", "import re\n", "import json\n", "\n", "# Support to access the remote target\n", "import devlib\n", "from env import TestEnv\n", "\n", "import trappy\n", "\n", "import socket\n", "\n", "from time import sleep\n", "\n", "import netifaces as ni" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Target Configuration" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Define your path to the ChromeOS installation folder\n", "CROS_BASE = \"/data/chromiumos\"" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Setup a target configuration\n", "my_conf = {\n", " \n", " # Target platform and board\n", " \"platform\" : 'linux',\n", " \n", " # Target board IP/MAC address\n", " \"host\" : '192.168.0.1',\n", " \n", " # Login credentials\n", " \"username\" : 'root',\n", " \"password\" : 'test0000',\n", " \n", " # Tools required by the experiments\n", " \"tools\" : [ 'trace-cmd' ], \n", " \n", " # FTrace events to collect for all the tests configuration which have\n", " # the \"ftrace\" flag enabled\n", " \"ftrace\" : {\n", " \"events\" : [\n", " \"cpu_frequency\",\n", " \"cpu_idle\",\n", " \"sched_switch\"\n", " ],\n", " \"buffsize\" : 10 * 1024,\n", " },\n", "}" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false, "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2016-04-27 11:06:35,781 INFO : Target - Using base path: /home/pippo/work/lisa\n", "2016-04-27 11:06:35,782 INFO : Target - Loading custom (inline) target configuration\n", "2016-04-27 11:06:35,782 DEBUG : Target - Target configuration {'username': 'root', 'ftrace': {'buffsize': 10240, 'events': ['cpu_frequency', 'cpu_idle', 'sched_switch']}, 'platform': 'linux', 'host': '192.168.0.1', 'board': 'oak', 'password': 'test0000', 'tools': ['trace-cmd']}\n", "2016-04-27 11:06:35,783 INFO : Target - Devlib modules to load: ['bl', 'cpufreq']\n", "2016-04-27 11:06:35,783 INFO : Target - Connecting linux target:\n", "2016-04-27 11:06:35,784 INFO : Target - username : root\n", "2016-04-27 11:06:35,785 INFO : Target - host : 192.168.0.1\n", "2016-04-27 11:06:35,785 INFO : Target - password : test0000\n", "2016-04-27 11:06:35,786 DEBUG : Target - Setup LINUX target...\n", "2016-04-27 11:06:35,795 DEBUG : Logging in root@192.168.0.1\n", "2016-04-27 11:06:37,318 DEBUG : id\n", "2016-04-27 11:06:37,727 DEBUG : if [ -e '/root/devlib-target/bin' ]; then echo 1; else echo 0; fi\n", "2016-04-27 11:06:38,133 DEBUG : ls -1 /root/devlib-target/bin\n", "2016-04-27 11:06:38,541 DEBUG : cat /proc/cpuinfo\n", "2016-04-27 11:06:38,851 DEBUG : Installing module bl\n", "2016-04-27 11:06:38,952 DEBUG : /root/devlib-target/bin/busybox uname -m\n", "2016-04-27 11:06:39,359 DEBUG : if [ -e '/sys/devices/system/cpu/cpufreq' ]; then echo 1; else echo 0; fi\n", "2016-04-27 11:06:39,665 DEBUG : Installing module cpufreq\n", "2016-04-27 11:06:39,666 DEBUG : Target - Checking target connection...\n", "2016-04-27 11:06:39,666 DEBUG : Target - Target info:\n", "2016-04-27 11:06:39,667 DEBUG : Target - ABI: arm64\n", "2016-04-27 11:06:39,668 DEBUG : Target - CPUs: CpuInfo(['A53', 'A53', 'A72', 'A72'])\n", "2016-04-27 11:06:39,668 DEBUG : Target - Clusters: [0, 0, 1, 1]\n", "2016-04-27 11:06:39,770 DEBUG : sudo -- sh -c 'mount -o remount,rw /'\n", "2016-04-27 11:06:40,574 INFO : Target - Initializing target workdir:\n", "2016-04-27 11:06:40,575 INFO : Target - /root/devlib-target\n", "2016-04-27 11:06:40,676 DEBUG : mkdir -p /root/devlib-target\n", "2016-04-27 11:06:41,085 DEBUG : mkdir -p /root/devlib-target/bin\n", "2016-04-27 11:06:41,393 DEBUG : /usr/bin/scp -r /home/pippo/work/lisa/libs/devlib/devlib/bin/arm64/busybox root@192.168.0.1:/root/devlib-target/bin/busybox\n", "2016-04-27 11:06:41,804 DEBUG : chmod a+x /root/devlib-target/bin/busybox\n", "2016-04-27 11:06:42,112 DEBUG : /usr/bin/scp -r /home/pippo/work/lisa/libs/devlib/devlib/bin/scripts/shutils root@192.168.0.1:/root/devlib-target/bin/shutils\n", "2016-04-27 11:06:42,360 DEBUG : chmod a+x /root/devlib-target/bin/shutils\n", "2016-04-27 11:06:42,668 DEBUG : /usr/bin/scp -r /home/pippo/work/lisa/tools/arm64/trace-cmd root@192.168.0.1:/root/devlib-target/bin/trace-cmd\n", "2016-04-27 11:06:42,976 DEBUG : chmod a+x /root/devlib-target/bin/trace-cmd\n", "2016-04-27 11:06:43,285 DEBUG : Target - Check for module [bl]...\n", "2016-04-27 11:06:43,286 DEBUG : Target - Check for module [cpufreq]...\n", "2016-04-27 11:06:43,287 INFO : Target - Topology:\n", "2016-04-27 11:06:43,287 INFO : Target - [[0, 1], [2, 3]]\n", "2016-04-27 11:06:43,389 DEBUG : sudo -- sh -c 'cat '\\''/sys/devices/system/cpu/online'\\'''\n", "2016-04-27 11:06:44,293 DEBUG : cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_frequencies\n", "2016-04-27 11:06:44,702 DEBUG : sudo -- sh -c 'cat '\\''/sys/devices/system/cpu/online'\\'''\n", "2016-04-27 11:06:45,606 DEBUG : cat /sys/devices/system/cpu/cpu2/cpufreq/scaling_available_frequencies\n", "2016-04-27 11:06:45,914 DEBUG : Platform - Trying to load default EM from /home/pippo/work/lisa/libs/utils/platforms/oak.json\n", "2016-04-27 11:06:45,915 DEBUG : Platform - Platform descriptor initialized\n", "{'nrg_model': None, 'clusters': {'big': [2, 3], 'little': [0, 1]}, 'cpus_count': 4, 'freqs': {'big': [507000, 702000, 1001000, 1209000, 1404000, 1612000, 1807000, 2106000], 'little': [507000, 702000, 1001000, 1105000, 1209000, 1300000, 1508000, 1703000]}, 'topology': [[0, 1], [2, 3]]}\n", "2016-04-27 11:06:45,932 DEBUG : /usr/bin/scp -r /home/pippo/work/lisa/libs/devlib/devlib/bin/arm64/trace-cmd root@192.168.0.1:/root/devlib-target/bin/trace-cmd\n", "2016-04-27 11:06:46,296 DEBUG : chmod a+x /root/devlib-target/bin/trace-cmd\n", "2016-04-27 11:06:46,704 DEBUG : cat /sys/kernel/debug/tracing/available_events\n", "2016-04-27 11:06:47,036 INFO : FTrace - Enabled tracepoints:\n", "2016-04-27 11:06:47,037 INFO : FTrace - cpu_frequency\n", "2016-04-27 11:06:47,038 INFO : FTrace - cpu_idle\n", "2016-04-27 11:06:47,038 INFO : FTrace - sched_switch\n", "2016-04-27 11:06:47,039 DEBUG : No RT-App workloads, skipping calibration\n", "2016-04-27 11:06:47,040 INFO : TestEnv - Set results folder to:\n", "2016-04-27 11:06:47,040 INFO : TestEnv - /home/pippo/work/lisa/results/20160427_110647\n", "2016-04-27 11:06:47,040 INFO : TestEnv - Experiment results available also in:\n", "2016-04-27 11:06:47,041 INFO : TestEnv - /home/pippo/work/lisa/results_latest\n" ] } ], "source": [ "# Initialize a test environment using:\n", "# the provided target configuration (my_conf)\n", "te = TestEnv(my_conf)\n", "target = te.target" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [], "source": [ "def get_host_ip():\n", " \"\"\"Returns the IP of the local host\"\"\"\n", " ifs = ni.interfaces()\n", " for interface in ifs:\n", " if interface == 'lo':\n", " continue\n", "\n", " addresses = ni.ifaddresses(interface)\n", " if addresses.has_key(socket.AF_INET):\n", " return addresses[socket.AF_INET][0]['addr']\n", "\n", "HOST_IP = get_host_ip()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Set ChromeOS paths" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "CROS_SDK_BIN_PATH = CROS_BASE + \"/chromium/tools/depot_tools/cros_sdk\"\n", "username = !id -un\n", "CROS_PATH = CROS_BASE + \"/chroot/home/\" + username[0]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Test-specific Parser Functions" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def parse_graphics_WebGLAquarium(results_dir):\n", " results_file = os.path.join(\n", " CROS_PATH,\n", " os.path.basename(results_dir),\n", " 'results-1-graphics_WebGLAquarium/graphics_WebGLAquarium/results/keyval'\n", " )\n", " \n", " data = {}\n", " with open(results_file) as data_file:\n", " for line in data_file:\n", " if line.strip():\n", " key, val = line.split('=')\n", " data[key] = float(val)\n", " \n", " return data" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "parse_results = {\n", " # Acquarium\n", " 'graphics_WebGLAquarium' : parse_graphics_WebGLAquarium\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Run a Test" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": false, "run_control": { "marked": false } }, "outputs": [], "source": [ "def CrosSdkSession(password):\n", " \"\"\"\n", " Create cros_sdk session. The user will be asked to type his password.\n", " \n", " :param password: host machine password\n", " :type password: str\n", " \"\"\"\n", " cros_sdk_session = Popen(['sudo -Sk {}'.format(CROS_SDK_BIN_PATH)],\n", " bufsize=1,\n", " stdin=PIPE,\n", " stdout=PIPE,\n", " stderr=PIPE,\n", " cwd=CROS_PATH,\n", " shell=True)\n", " cros_sdk_session.stdin.write(password)\n", " cros_sdk_session.stdin.write('\\n')\n", " return cros_sdk_session\n", "\n", "def test_that(password, te, test, pwr_time_s, get_token=False):\n", " \"\"\"\n", " Run a specific test using the test_that command.\n", " \n", " :param password: host machine password\n", " :type password: str\n", " \n", " :param te: Test Environment object\n", " :type te: env.TestEnv\n", " \n", " :param test: name of the test to be run\n", " :type test: str\n", " \n", " :param pwr_time_s: power measurement duration in seconds\n", " :type pwr_time_s: int\n", " \n", " :param get_token: if True wait for token before collecting traces\n", " :type get_token: bool\n", " \"\"\"\n", " \n", " results_dir = \"~/results-dir\"\n", " pwr_file = \"~/power.txt\"\n", " \n", " test_cmd = 'test_that -b oak {} --results_dir {} {}\\n'.format(te.ip,\n", " results_dir,\n", " test)\n", " pwr_cmd = 'dut-control -t {} -y dvfs1_mw dvfs2_mw > {}\\n'.format(pwr_time_s,\n", " pwr_file)\n", " \n", " # Create cros_sdk session\n", " cros_sdk_session = CrosSdkSession(password)\n", " \n", " logging.info('#### Start %s execution', test)\n", " cros_sdk_session.stdin.write(test_cmd)\n", " \n", " if get_token:\n", " # Setup socket to get token from target\n", " sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)\n", " sock.bind((HOST_IP, 1234))\n", " logging.debug(' Waiting for token...')\n", " while True:\n", " data, addr = sock.recvfrom(32)\n", " if addr[0] == te.ip and data == \"POWER\":\n", " break\n", " sock.close() \n", " logging.debug(' Token received....')\n", " \n", " logging.debug(' Start trace collection')\n", " # Check if trace events need to be collected\n", " if te.ftrace:\n", " te.ftrace.start()\n", " \n", " sleep(5)\n", " # Start measuring power\n", " cros_sdk_session.stdin.write(pwr_cmd)\n", " \n", " # communicate will close the session when the command terminates\n", " cros_sdk_session.communicate()\n", " logging.info('#### Completed %s execution', test)\n", " \n", " if te.ftrace:\n", " te.ftrace.stop()\n", " te.ftrace.get_trace(os.path.join(te.res_dir, 'trace.dat'))\n", " \n", " # Parse results using test-specific parser\n", " results = parse_results[test](results_dir)\n", " \n", " # Copy results to our Test Environment results directory\n", " with open(os.path.join(te.res_dir, 'results.json'), 'w') as outfile:\n", " json.dumps(results, outfile)\n", " \n", " return {\n", " \"results\" : results,\n", " \"pwr_file\" : pwr_file\n", " }" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": false, "run_control": { "marked": false } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "········\n" ] } ], "source": [ "# ask user for host password\n", "password = getpass.getpass()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": false, "run_control": { "marked": false } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2016-04-27 11:06:53,601 INFO : #### Start graphics_WebGLAquarium execution\n", "2016-04-27 11:06:53,603 DEBUG : Waiting for token...\n", "2016-04-27 11:07:23,040 DEBUG : Token received....\n", "2016-04-27 11:07:23,041 DEBUG : Start trace collection\n", "2016-04-27 11:07:23,142 DEBUG : sudo -- sh -c 'echo 10240 > '\\''/sys/kernel/debug/tracing/buffer_size_kb'\\'''\n", "2016-04-27 11:07:23,845 DEBUG : sudo -- sh -c 'cat '\\''/sys/kernel/debug/tracing/buffer_size_kb'\\'''\n", "2016-04-27 11:07:24,750 DEBUG : sudo -- sh -c '/root/devlib-target/bin/trace-cmd reset'\n", "2016-04-27 11:07:26,170 DEBUG : sudo -- sh -c '/root/devlib-target/bin/trace-cmd start -e cpu_frequency -e cpu_idle -e sched_switch'\n", "2016-04-27 11:07:27,539 DEBUG : sudo -- sh -c 'echo TRACE_MARKER_START > '\\''/sys/kernel/debug/tracing/trace_marker'\\'''\n", "2016-04-27 11:07:28,141 DEBUG : Trace CPUFreq frequencies\n", "2016-04-27 11:07:28,242 DEBUG : sudo -- sh -c '/root/devlib-target/bin/shutils cpufreq_trace_all_frequencies'\n", "2016-04-27 11:08:45,869 INFO : #### Completed graphics_WebGLAquarium execution\n", "2016-04-27 11:08:45,870 DEBUG : Trace CPUFreq frequencies\n", "2016-04-27 11:08:45,971 DEBUG : sudo -- sh -c '/root/devlib-target/bin/shutils cpufreq_trace_all_frequencies'\n", "2016-04-27 11:08:46,875 DEBUG : sudo -- sh -c 'echo TRACE_MARKER_STOP > '\\''/sys/kernel/debug/tracing/trace_marker'\\'''\n", "2016-04-27 11:08:47,578 DEBUG : sudo -- sh -c '/root/devlib-target/bin/trace-cmd stop'\n", "2016-04-27 11:08:48,482 DEBUG : sudo -- sh -c '/root/devlib-target/bin/trace-cmd extract -o /root/devlib-target/trace.dat'\n", "2016-04-27 11:08:50,337 DEBUG : /usr/bin/scp -r root@192.168.0.1:/root/devlib-target/trace.dat /home/pippo/work/lisa/results/20160427_110647/trace.dat\n" ] } ], "source": [ "# Run Acquarium and collect results\n", "acquarium_res = test_that(password,\n", " te,\n", " 'graphics_WebGLAquarium',\n", " 10,\n", " get_token=True)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Values
avg_render_time_0050_fishes{perf}0.007754
std_render_time_0050_fishes{perf}0.003022
avg_render_time_1000_fishes{perf}0.015975
meminfo_MemUsed{perf}1052424.000000
std_interframe_time_1000_fishes{perf}0.036034
avg_fps_1000_fishes{perf}38.576854
std_interframe_time_0050_fishes{perf}0.029933
std_render_time_1000_fishes{perf}0.002286
meminfo_SwapUsed{perf}0.000000
avg_interframe_time_1000_fishes{perf}0.025922
avg_interframe_time_0050_fishes{perf}0.017457
avg_fps_0050_fishes{perf}57.282652
\n", "
" ], "text/plain": [ " Values\n", "avg_render_time_0050_fishes{perf} 0.007754\n", "std_render_time_0050_fishes{perf} 0.003022\n", "avg_render_time_1000_fishes{perf} 0.015975\n", "meminfo_MemUsed{perf} 1052424.000000\n", "std_interframe_time_1000_fishes{perf} 0.036034\n", "avg_fps_1000_fishes{perf} 38.576854\n", "std_interframe_time_0050_fishes{perf} 0.029933\n", "std_render_time_1000_fishes{perf} 0.002286\n", "meminfo_SwapUsed{perf} 0.000000\n", "avg_interframe_time_1000_fishes{perf} 0.025922\n", "avg_interframe_time_0050_fishes{perf} 0.017457\n", "avg_fps_0050_fishes{perf} 57.282652" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Print Acquarium results\n", "df = pd.DataFrame.from_dict(acquarium_res['results'], orient=\"index\")\n", "df.columns = ['Values']\n", "df" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Plot trace data using TRAPpy" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false }, "outputs": [], "source": [ "if te.ftrace:\n", " trappy.plotter.plot_trace(te.res_dir)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Compute Energy" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Energy [mJ]
Total4573.172353
LITTLE Cluster1574.512282
big Cluster2998.660071
\n", "
" ], "text/plain": [ " Energy [mJ]\n", "Total 4573.172353\n", "LITTLE Cluster 1574.512282\n", "big Cluster 2998.660071" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Collect SERVO trace\n", "servo_trace_file = os.path.join(CROS_PATH,\n", " os.path.basename(acquarium_res['pwr_file']))\n", "time = []\n", "dvfs1_wm = []\n", "dvfs2_wm = []\n", "with open(servo_trace_file, 'r') as f:\n", " for l in f:\n", " if l.startswith(\"@@\"):\n", " continue\n", "\n", " info = re.split(r'[ :]+', l)\n", " if info[1] == \"dvfs1_mw\":\n", " time.append(float(info[0]))\n", " dvfs1_wm.append(float(info[2]))\n", " else:\n", " dvfs2_wm.append(float(info[2]))\n", "\n", "# Create dataframes for power data from SERVO board\n", "big_pwr = pd.DataFrame(dvfs1_wm, index=time, columns=['Power'])\n", "little_pwr = pd.DataFrame(dvfs2_wm, index=time, columns=['Power'])\n", "\n", "x = big_pwr.index.get_values()\n", "y = big_pwr.Power.get_values()\n", "bnrg = integrate.simps(y, x=x)\n", "\n", "x = little_pwr.index.get_values()\n", "y = little_pwr.Power.get_values()\n", "lnrg = integrate.simps(y, x=x)\n", "results = {\n", " 'big Cluster' : bnrg,\n", " 'LITTLE Cluster' : lnrg,\n", " 'Total' : bnrg + lnrg,\n", "}\n", "\n", "df = pd.DataFrame.from_dict(results, orient=\"index\")\n", "df.columns = ['Energy [mJ]']\n", "df" ] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.9" }, "toc": { "toc_cell": false, "toc_number_sections": true, "toc_threshold": 6, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 0 }