1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.targetprep; 18 19 import com.android.tradefed.build.IBuildInfo; 20 import com.android.tradefed.config.Option; 21 import com.android.tradefed.config.OptionClass; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.util.CommandResult; 26 import com.android.tradefed.util.CommandStatus; 27 import com.android.tradefed.util.FileUtil; 28 import com.android.tradefed.util.IRunUtil; 29 import com.android.tradefed.util.RunUtil; 30 import com.android.tradefed.util.StreamUtil; 31 32 import org.json.JSONException; 33 import org.json.JSONObject; 34 35 import java.io.File; 36 import java.io.InputStream; 37 import java.io.IOException; 38 import java.util.Arrays; 39 import java.util.Collection; 40 import java.util.NoSuchElementException; 41 import java.util.TreeSet; 42 43 /** 44 * Sets up a Python virtualenv on the host and installs packages. To activate it, the working 45 * directory is changed to the root of the virtualenv. 46 * 47 * This's a fork of PythonVirtualenvPreparer and is forked in order to simplify the change 48 * deployment process and reduce the deployment time, which are critical for VTS services. 49 * That means changes here will be upstreamed gradually. 50 */ 51 @OptionClass(alias = "python-venv") 52 public class VtsPythonVirtualenvPreparer implements ITargetPreparer, ITargetCleaner { 53 54 private static final String PIP = "pip"; 55 private static final String PATH = "PATH"; 56 private static final String OS_NAME = "os.name"; 57 private static final String WINDOWS = "Windows"; 58 private static final String LOCAL_PYPI_PATH_ENV_VAR_NAME = "VTS_PYPI_PATH"; 59 private static final String VENDOR_TEST_CONFIG_FILE_PATH = 60 "/config/google-tradefed-vts-config.config"; 61 private static final String LOCAL_PYPI_PATH_KEY = "pypi_packages_path"; 62 protected static final String PYTHONPATH = "PYTHONPATH"; 63 protected static final String VIRTUAL_ENV_PATH = "VIRTUALENVPATH"; 64 private static final int BASE_TIMEOUT = 1000 * 60; 65 private static final String[] DEFAULT_DEP_MODULES = {"enum", "future", "futures", 66 "google-api-python-client", "httplib2", "oauth2client", "protobuf", "requests"}; 67 68 @Option(name = "venv-dir", description = "path of an existing virtualenv to use") 69 private File mVenvDir = null; 70 71 @Option(name = "requirements-file", description = "pip-formatted requirements file") 72 private File mRequirementsFile = null; 73 74 @Option(name = "script-file", description = "scripts which need to be executed in advance") 75 private Collection<String> mScriptFiles = new TreeSet<>(); 76 77 @Option(name = "dep-module", description = "modules which need to be installed by pip") 78 private Collection<String> mDepModules = new TreeSet<>(Arrays.asList(DEFAULT_DEP_MODULES)); 79 80 IRunUtil mRunUtil = new RunUtil(); 81 String mPip = PIP; 82 String mLocalPypiPath = null; 83 84 /** 85 * {@inheritDoc} 86 */ 87 @Override setUp(ITestDevice device, IBuildInfo buildInfo)88 public void setUp(ITestDevice device, IBuildInfo buildInfo) 89 throws TargetSetupError, BuildError, DeviceNotAvailableException { 90 startVirtualenv(buildInfo); 91 setLocalPypiPath(); 92 installDeps(buildInfo); 93 } 94 95 /** 96 * {@inheritDoc} 97 */ 98 @Override tearDown(ITestDevice device, IBuildInfo buildInfo, Throwable e)99 public void tearDown(ITestDevice device, IBuildInfo buildInfo, Throwable e) 100 throws DeviceNotAvailableException { 101 if (mVenvDir != null) { 102 FileUtil.recursiveDelete(mVenvDir); 103 CLog.i("Deleted the virtual env's temp working dir, %s.", mVenvDir); 104 mVenvDir = null; 105 } 106 } 107 108 /** 109 * This method sets mLocalPypiPath, the local PyPI package directory to 110 * install python packages from in the installDeps method. 111 * 112 * @throws IOException 113 * @throws JSONException 114 */ setLocalPypiPath()115 protected void setLocalPypiPath() throws RuntimeException { 116 CLog.i("Loading vendor test config %s", VENDOR_TEST_CONFIG_FILE_PATH); 117 InputStream config = getClass().getResourceAsStream(VENDOR_TEST_CONFIG_FILE_PATH); 118 119 // First try to load local PyPI directory path from vendor config file 120 if (config != null) { 121 try { 122 String content = StreamUtil.getStringFromStream(config); 123 CLog.i("Loaded vendor test config %s", content); 124 if (content != null) { 125 JSONObject vendorConfigJson = new JSONObject(content); 126 try { 127 String pypiPath = vendorConfigJson.getString(LOCAL_PYPI_PATH_KEY); 128 if (pypiPath.length() > 0 && dirExistsAndHaveReadAccess(pypiPath)) { 129 mLocalPypiPath = pypiPath; 130 CLog.i(String.format( 131 "Loaded %s: %s", LOCAL_PYPI_PATH_KEY, mLocalPypiPath)); 132 } 133 } catch (NoSuchElementException e) { 134 CLog.i("Vendor test config file does not define %s", LOCAL_PYPI_PATH_KEY); 135 } 136 } 137 } catch (IOException e) { 138 throw new RuntimeException("Failed to read vendor config json file"); 139 } catch (JSONException e) { 140 throw new RuntimeException("Failed to parse vendor config json data"); 141 } 142 } else { 143 CLog.i("Vendor test config file %s does not exist", VENDOR_TEST_CONFIG_FILE_PATH); 144 } 145 146 // If loading path from vendor config file is unsuccessful, 147 // check local pypi path defined by LOCAL_PYPI_PATH_ENV_VAR_NAME 148 if (mLocalPypiPath == null) { 149 CLog.i("Checking whether local pypi packages directory exists"); 150 String pypiPath = System.getenv(LOCAL_PYPI_PATH_ENV_VAR_NAME); 151 if (pypiPath == null) { 152 CLog.i("Local pypi packages directory not specified by env var %s", 153 LOCAL_PYPI_PATH_ENV_VAR_NAME); 154 } else if (dirExistsAndHaveReadAccess(pypiPath)) { 155 mLocalPypiPath = pypiPath; 156 CLog.i("Set local pypi packages directory to %s", pypiPath); 157 } 158 } 159 160 if (mLocalPypiPath == null) { 161 CLog.i("Failed to set local pypi packages path. Therefore internet connection to " 162 + "https://pypi.python.org/simple/ must be available to run VTS tests."); 163 } 164 } 165 166 /** 167 * This method returns whether the given path is a dir that exists and the user has read access. 168 */ dirExistsAndHaveReadAccess(String path)169 private boolean dirExistsAndHaveReadAccess(String path) { 170 File pathDir = new File(path); 171 if (!pathDir.exists() || !pathDir.isDirectory()) { 172 CLog.i("Directory %s does not exist.", pathDir); 173 return false; 174 } 175 176 if (!isOnWindows()) { 177 CommandResult c = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, "ls", path); 178 if (c.getStatus() != CommandStatus.SUCCESS) { 179 CLog.i(String.format("Failed to read dir: %s. Result %s. stdout: %s, stderr: %s", 180 path, c.getStatus(), c.getStdout(), c.getStderr())); 181 return false; 182 } 183 return true; 184 } else { 185 try { 186 String[] pathDirList = pathDir.list(); 187 if (pathDirList == null) { 188 CLog.i("Failed to read dir: %s. Please check access permission.", pathDir); 189 return false; 190 } 191 } catch (SecurityException e) { 192 CLog.i(String.format( 193 "Failed to read dir %s with SecurityException %s", pathDir, e)); 194 return false; 195 } 196 return true; 197 } 198 } 199 installDeps(IBuildInfo buildInfo)200 protected void installDeps(IBuildInfo buildInfo) throws TargetSetupError { 201 boolean hasDependencies = false; 202 if (!mScriptFiles.isEmpty()) { 203 for (String scriptFile : mScriptFiles) { 204 CLog.i("Attempting to execute a script, %s", scriptFile); 205 CommandResult c = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, scriptFile); 206 if (c.getStatus() != CommandStatus.SUCCESS) { 207 CLog.e("Executing script %s failed", scriptFile); 208 throw new TargetSetupError("Failed to source a script"); 209 } 210 } 211 } 212 if (mRequirementsFile != null) { 213 CommandResult c = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, mPip, 214 "install", "-r", mRequirementsFile.getAbsolutePath()); 215 if (c.getStatus() != CommandStatus.SUCCESS) { 216 CLog.e("Installing dependencies from %s failed", 217 mRequirementsFile.getAbsolutePath()); 218 throw new TargetSetupError("Failed to install dependencies with pip"); 219 } 220 hasDependencies = true; 221 } 222 if (!mDepModules.isEmpty()) { 223 for (String dep : mDepModules) { 224 CommandResult result = null; 225 if (mLocalPypiPath != null) { 226 CLog.i("Attempting installation of %s from local directory", dep); 227 result = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, mPip, "install", dep, 228 "--no-index", "--find-links=" + mLocalPypiPath); 229 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 230 result.getStdout(), result.getStderr())); 231 if (result.getStatus() != CommandStatus.SUCCESS) { 232 CLog.e(String.format("Installing %s from %s failed", dep, mLocalPypiPath)); 233 } 234 } 235 if (mLocalPypiPath == null || result.getStatus() != CommandStatus.SUCCESS) { 236 CLog.i("Attempting installation of %s from PyPI", dep); 237 result = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, mPip, "install", dep); 238 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 239 result.getStdout(), result.getStderr())); 240 if (result.getStatus() != CommandStatus.SUCCESS) { 241 CLog.e("Installing %s from PyPI failed.", dep); 242 CLog.i("Attempting to upgrade %s", dep); 243 result = mRunUtil.runTimedCmd( 244 BASE_TIMEOUT * 5, mPip, "install", "--upgrade", dep); 245 if (result.getStatus() != CommandStatus.SUCCESS) { 246 throw new TargetSetupError(String.format( 247 "Failed to install dependencies with pip. " 248 + "Result %s. stdout: %s, stderr: %s", 249 result.getStatus(), result.getStdout(), result.getStderr())); 250 } else { 251 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", 252 result.getStatus(), result.getStdout(), result.getStderr())); 253 } 254 } 255 } 256 hasDependencies = true; 257 } 258 } 259 if (!hasDependencies) { 260 CLog.i("No dependencies to install"); 261 } else { 262 // make the install directory of new packages available to other classes that 263 // receive the build 264 buildInfo.setFile(PYTHONPATH, new File(mVenvDir, 265 "local/lib/python2.7/site-packages"), 266 buildInfo.getBuildId()); 267 } 268 } 269 startVirtualenv(IBuildInfo buildInfo)270 protected void startVirtualenv(IBuildInfo buildInfo) throws TargetSetupError { 271 if (mVenvDir != null) { 272 CLog.i("Using existing virtualenv based at %s", mVenvDir.getAbsolutePath()); 273 activate(); 274 return; 275 } 276 try { 277 mVenvDir = buildInfo.getFile(VIRTUAL_ENV_PATH); 278 if (mVenvDir == null) { 279 mVenvDir = FileUtil.createTempDir(buildInfo.getTestTag() + "-virtualenv"); 280 } 281 String virtualEnvPath = mVenvDir.getAbsolutePath(); 282 mRunUtil.runTimedCmd(BASE_TIMEOUT, "virtualenv", virtualEnvPath); 283 CLog.i(VIRTUAL_ENV_PATH + " = " + virtualEnvPath + "\n"); 284 buildInfo.setFile(VIRTUAL_ENV_PATH, new File(virtualEnvPath), 285 buildInfo.getBuildId()); 286 activate(); 287 } catch (IOException e) { 288 CLog.e("Failed to create temp directory for virtualenv"); 289 throw new TargetSetupError("Error creating virtualenv", e); 290 } 291 } 292 addDepModule(String module)293 protected void addDepModule(String module) { 294 mDepModules.add(module); 295 } 296 setRequirementsFile(File f)297 protected void setRequirementsFile(File f) { 298 mRequirementsFile = f; 299 } 300 301 /** 302 * This method returns whether the OS is Windows. 303 */ isOnWindows()304 private static boolean isOnWindows() { 305 return System.getProperty(OS_NAME).contains(WINDOWS); 306 } 307 activate()308 private void activate() { 309 File binDir = new File(mVenvDir, isOnWindows() ? "Scripts" : "bin"); 310 mRunUtil.setWorkingDir(binDir); 311 String path = System.getenv(PATH); 312 mRunUtil.setEnvVariable(PATH, binDir + File.pathSeparator + path); 313 File pipFile = new File(binDir, PIP); 314 pipFile.setExecutable(true); 315 mPip = pipFile.getAbsolutePath(); 316 } 317 } 318