1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.targetprep; 18 19 import com.android.tradefed.build.IBuildInfo; 20 import com.android.tradefed.config.Option; 21 import com.android.tradefed.config.OptionClass; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.util.CommandResult; 26 import com.android.tradefed.util.CommandStatus; 27 import com.android.tradefed.util.FileUtil; 28 import com.android.tradefed.util.IRunUtil; 29 import com.android.tradefed.util.RunUtil; 30 import com.android.tradefed.util.StreamUtil; 31 import com.android.tradefed.util.VtsVendorConfigFileUtil; 32 33 import org.json.JSONException; 34 import org.json.JSONObject; 35 36 import java.io.File; 37 import java.io.InputStream; 38 import java.nio.file.FileVisitResult; 39 import java.nio.file.Files; 40 import java.nio.file.Path; 41 import java.nio.file.SimpleFileVisitor; 42 import java.nio.file.attribute.BasicFileAttributes; 43 import java.io.IOException; 44 import java.security.MessageDigest; 45 import java.security.NoSuchAlgorithmException; 46 import java.util.Arrays; 47 import java.util.Collection; 48 import java.util.NoSuchElementException; 49 import java.util.TreeSet; 50 51 /** 52 * Sets up a Python virtualenv on the host and installs packages. To activate it, the working 53 * directory is changed to the root of the virtualenv. 54 * 55 * This's a fork of PythonVirtualenvPreparer and is forked in order to simplify the change 56 * deployment process and reduce the deployment time, which are critical for VTS services. 57 * That means changes here will be upstreamed gradually. 58 */ 59 @OptionClass(alias = "python-venv") 60 public class VtsPythonVirtualenvPreparer implements ITargetPreparer, ITargetCleaner { 61 62 private static final String PIP = "pip"; 63 private static final String PATH = "PATH"; 64 private static final String OS_NAME = "os.name"; 65 private static final String WINDOWS = "Windows"; 66 private static final String LOCAL_PYPI_PATH_ENV_VAR_NAME = "VTS_PYPI_PATH"; 67 private static final String LOCAL_PYPI_PATH_KEY = "pypi_packages_path"; 68 protected static final String PYTHONPATH = "PYTHONPATH"; 69 protected static final String VIRTUAL_ENV_PATH = "VIRTUALENVPATH"; 70 private static final int BASE_TIMEOUT = 1000 * 60; 71 private static final String[] DEFAULT_DEP_MODULES = {"enum", "future", "futures", 72 "google-api-python-client", "httplib2", "oauth2client", "protobuf", "requests"}; 73 74 @Option(name = "venv-dir", description = "path of an existing virtualenv to use") 75 private File mVenvDir = null; 76 77 @Option(name = "requirements-file", description = "pip-formatted requirements file") 78 private File mRequirementsFile = null; 79 80 @Option(name = "script-file", description = "scripts which need to be executed in advance") 81 private Collection<String> mScriptFiles = new TreeSet<>(); 82 83 @Option(name = "dep-module", description = "modules which need to be installed by pip") 84 private Collection<String> mDepModules = new TreeSet<>(Arrays.asList(DEFAULT_DEP_MODULES)); 85 86 IBuildInfo mBuildInfo = null; 87 IRunUtil mRunUtil = new RunUtil(); 88 String mPip = PIP; 89 String mLocalPypiPath = null; 90 91 /** 92 * {@inheritDoc} 93 */ 94 @Override setUp(ITestDevice device, IBuildInfo buildInfo)95 public void setUp(ITestDevice device, IBuildInfo buildInfo) 96 throws TargetSetupError, BuildError, DeviceNotAvailableException { 97 mBuildInfo = buildInfo; 98 startVirtualenv(buildInfo); 99 setLocalPypiPath(); 100 installDeps(buildInfo); 101 } 102 103 /** 104 * {@inheritDoc} 105 */ 106 @Override tearDown(ITestDevice device, IBuildInfo buildInfo, Throwable e)107 public void tearDown(ITestDevice device, IBuildInfo buildInfo, Throwable e) 108 throws DeviceNotAvailableException { 109 if (mVenvDir != null) { 110 try { 111 recursiveDelete(mVenvDir.toPath()); 112 CLog.i("Deleted the virtual env's temp working dir, %s.", mVenvDir); 113 } catch (IOException exception) { 114 CLog.e("Failed to delete %s: %s", mVenvDir, exception); 115 } 116 mVenvDir = null; 117 } 118 } 119 120 /** 121 * This method sets mLocalPypiPath, the local PyPI package directory to 122 * install python packages from in the installDeps method. 123 * 124 * @throws IOException 125 * @throws JSONException 126 */ setLocalPypiPath()127 protected void setLocalPypiPath() throws RuntimeException { 128 VtsVendorConfigFileUtil configReader = new VtsVendorConfigFileUtil(); 129 if (configReader.LoadVendorConfig(mBuildInfo)) { 130 // First try to load local PyPI directory path from vendor config file 131 try { 132 String pypiPath = configReader.GetVendorConfigVariable(LOCAL_PYPI_PATH_KEY); 133 if (pypiPath.length() > 0 && dirExistsAndHaveReadAccess(pypiPath)) { 134 mLocalPypiPath = pypiPath; 135 CLog.i(String.format("Loaded %s: %s", LOCAL_PYPI_PATH_KEY, mLocalPypiPath)); 136 } 137 } catch (NoSuchElementException e) { 138 /* continue */ 139 } 140 } 141 142 // If loading path from vendor config file is unsuccessful, 143 // check local pypi path defined by LOCAL_PYPI_PATH_ENV_VAR_NAME 144 if (mLocalPypiPath == null) { 145 CLog.i("Checking whether local pypi packages directory exists"); 146 String pypiPath = System.getenv(LOCAL_PYPI_PATH_ENV_VAR_NAME); 147 if (pypiPath == null) { 148 CLog.i("Local pypi packages directory not specified by env var %s", 149 LOCAL_PYPI_PATH_ENV_VAR_NAME); 150 } else if (dirExistsAndHaveReadAccess(pypiPath)) { 151 mLocalPypiPath = pypiPath; 152 CLog.i("Set local pypi packages directory to %s", pypiPath); 153 } 154 } 155 156 if (mLocalPypiPath == null) { 157 CLog.i("Failed to set local pypi packages path. Therefore internet connection to " 158 + "https://pypi.python.org/simple/ must be available to run VTS tests."); 159 } 160 } 161 162 /** 163 * This method returns whether the given path is a dir that exists and the user has read access. 164 */ dirExistsAndHaveReadAccess(String path)165 private boolean dirExistsAndHaveReadAccess(String path) { 166 File pathDir = new File(path); 167 if (!pathDir.exists() || !pathDir.isDirectory()) { 168 CLog.i("Directory %s does not exist.", pathDir); 169 return false; 170 } 171 172 if (!isOnWindows()) { 173 CommandResult c = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, "ls", path); 174 if (c.getStatus() != CommandStatus.SUCCESS) { 175 CLog.i(String.format("Failed to read dir: %s. Result %s. stdout: %s, stderr: %s", 176 path, c.getStatus(), c.getStdout(), c.getStderr())); 177 return false; 178 } 179 return true; 180 } else { 181 try { 182 String[] pathDirList = pathDir.list(); 183 if (pathDirList == null) { 184 CLog.i("Failed to read dir: %s. Please check access permission.", pathDir); 185 return false; 186 } 187 } catch (SecurityException e) { 188 CLog.i(String.format( 189 "Failed to read dir %s with SecurityException %s", pathDir, e)); 190 return false; 191 } 192 return true; 193 } 194 } 195 installDeps(IBuildInfo buildInfo)196 protected void installDeps(IBuildInfo buildInfo) throws TargetSetupError { 197 boolean hasDependencies = false; 198 if (!mScriptFiles.isEmpty()) { 199 for (String scriptFile : mScriptFiles) { 200 CLog.i("Attempting to execute a script, %s", scriptFile); 201 CommandResult c = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, scriptFile); 202 if (c.getStatus() != CommandStatus.SUCCESS) { 203 CLog.e("Executing script %s failed", scriptFile); 204 throw new TargetSetupError("Failed to source a script"); 205 } 206 } 207 } 208 if (mRequirementsFile != null) { 209 CommandResult c = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, mPip, 210 "install", "-r", mRequirementsFile.getAbsolutePath()); 211 if (c.getStatus() != CommandStatus.SUCCESS) { 212 CLog.e("Installing dependencies from %s failed", 213 mRequirementsFile.getAbsolutePath()); 214 throw new TargetSetupError("Failed to install dependencies with pip"); 215 } 216 hasDependencies = true; 217 } 218 if (!mDepModules.isEmpty()) { 219 for (String dep : mDepModules) { 220 CommandResult result = null; 221 if (mLocalPypiPath != null) { 222 CLog.i("Attempting installation of %s from local directory", dep); 223 result = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, mPip, "install", dep, 224 "--no-index", "--find-links=" + mLocalPypiPath); 225 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 226 result.getStdout(), result.getStderr())); 227 if (result.getStatus() != CommandStatus.SUCCESS) { 228 CLog.e(String.format("Installing %s from %s failed", dep, mLocalPypiPath)); 229 } 230 } 231 if (mLocalPypiPath == null || result.getStatus() != CommandStatus.SUCCESS) { 232 CLog.i("Attempting installation of %s from PyPI", dep); 233 result = mRunUtil.runTimedCmd(BASE_TIMEOUT * 5, mPip, "install", dep); 234 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", result.getStatus(), 235 result.getStdout(), result.getStderr())); 236 if (result.getStatus() != CommandStatus.SUCCESS) { 237 CLog.e("Installing %s from PyPI failed.", dep); 238 CLog.i("Attempting to upgrade %s", dep); 239 result = mRunUtil.runTimedCmd( 240 BASE_TIMEOUT * 5, mPip, "install", "--upgrade", dep); 241 if (result.getStatus() != CommandStatus.SUCCESS) { 242 throw new TargetSetupError(String.format( 243 "Failed to install dependencies with pip. " 244 + "Result %s. stdout: %s, stderr: %s", 245 result.getStatus(), result.getStdout(), result.getStderr())); 246 } else { 247 CLog.i(String.format("Result %s. stdout: %s, stderr: %s", 248 result.getStatus(), result.getStdout(), result.getStderr())); 249 } 250 } 251 } 252 hasDependencies = true; 253 } 254 } 255 if (!hasDependencies) { 256 CLog.i("No dependencies to install"); 257 } else { 258 // make the install directory of new packages available to other classes that 259 // receive the build 260 buildInfo.setFile(PYTHONPATH, new File(mVenvDir, 261 "local/lib/python2.7/site-packages"), 262 buildInfo.getBuildId()); 263 } 264 } 265 startVirtualenv(IBuildInfo buildInfo)266 protected void startVirtualenv(IBuildInfo buildInfo) throws TargetSetupError { 267 if (mVenvDir != null) { 268 CLog.i("Using existing virtualenv based at %s", mVenvDir.getAbsolutePath()); 269 activate(); 270 return; 271 } 272 try { 273 mVenvDir = buildInfo.getFile(VIRTUAL_ENV_PATH); 274 if (mVenvDir == null) { 275 mVenvDir = FileUtil.createTempDir(getMD5(buildInfo.getTestTag()) + "-virtualenv"); 276 } 277 String virtualEnvPath = mVenvDir.getAbsolutePath(); 278 CommandResult c = mRunUtil.runTimedCmd(BASE_TIMEOUT, "virtualenv", virtualEnvPath); 279 if (c.getStatus() != CommandStatus.SUCCESS) { 280 CLog.e(String.format("Failed to create virtualenv with : %s.", virtualEnvPath)); 281 throw new TargetSetupError("Failed to create virtualenv"); 282 } 283 CLog.i(VIRTUAL_ENV_PATH + " = " + virtualEnvPath + "\n"); 284 buildInfo.setFile(VIRTUAL_ENV_PATH, new File(virtualEnvPath), 285 buildInfo.getBuildId()); 286 activate(); 287 } catch (IOException | RuntimeException e) { 288 CLog.e("Failed to create temp directory for virtualenv"); 289 throw new TargetSetupError("Error creating virtualenv", e); 290 } 291 } 292 293 /** 294 * This method returns a MD5 hash string for the given string. 295 */ getMD5(String str)296 private String getMD5(String str) throws RuntimeException { 297 try { 298 java.security.MessageDigest md = java.security.MessageDigest.getInstance("MD5"); 299 byte[] array = md.digest(str.getBytes()); 300 StringBuffer sb = new StringBuffer(); 301 for (int i = 0; i < array.length; ++i) { 302 sb.append(Integer.toHexString((array[i] & 0xFF) | 0x100).substring(1, 3)); 303 } 304 return sb.toString(); 305 } catch (java.security.NoSuchAlgorithmException e) { 306 throw new RuntimeException("Error generating MD5 hash.", e); 307 } 308 } 309 addDepModule(String module)310 protected void addDepModule(String module) { 311 mDepModules.add(module); 312 } 313 setRequirementsFile(File f)314 protected void setRequirementsFile(File f) { 315 mRequirementsFile = f; 316 } 317 318 /** 319 * This method recursively deletes a file tree without following symbolic links. 320 * 321 * @param rootPath the path to delete. 322 * @throws IOException if fails to traverse or delete the files. 323 */ recursiveDelete(Path rootPath)324 private static void recursiveDelete(Path rootPath) throws IOException { 325 Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>() { 326 @Override 327 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) 328 throws IOException { 329 Files.delete(file); 330 return FileVisitResult.CONTINUE; 331 } 332 @Override 333 public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { 334 if (e != null) { 335 throw e; 336 } 337 Files.delete(dir); 338 return FileVisitResult.CONTINUE; 339 } 340 }); 341 } 342 343 /** 344 * This method returns whether the OS is Windows. 345 */ isOnWindows()346 private static boolean isOnWindows() { 347 return System.getProperty(OS_NAME).contains(WINDOWS); 348 } 349 activate()350 private void activate() { 351 File binDir = new File(mVenvDir, isOnWindows() ? "Scripts" : "bin"); 352 mRunUtil.setWorkingDir(binDir); 353 String path = System.getenv(PATH); 354 mRunUtil.setEnvVariable(PATH, binDir + File.pathSeparator + path); 355 File pipFile = new File(binDir, PIP); 356 pipFile.setExecutable(true); 357 mPip = pipFile.getAbsolutePath(); 358 } 359 } 360