1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.util.avd; 17 18 import static com.android.tradefed.util.avd.HostOrchestratorClient.Cvd; 19 import static com.android.tradefed.util.avd.HostOrchestratorClient.ErrorResponseException; 20 import static com.android.tradefed.util.avd.HostOrchestratorClient.HoHttpClient; 21 import static com.android.tradefed.util.avd.HostOrchestratorClient.IHoHttpClient; 22 import static com.android.tradefed.util.avd.HostOrchestratorClient.ListCvdsResponse; 23 import static com.android.tradefed.util.avd.HostOrchestratorClient.Operation; 24 import static com.android.tradefed.util.avd.HostOrchestratorClient.buildCreateBugreportRequest; 25 import static com.android.tradefed.util.avd.HostOrchestratorClient.buildGetOperationRequest; 26 import static com.android.tradefed.util.avd.HostOrchestratorClient.buildGetOperationResultRequest; 27 import static com.android.tradefed.util.avd.HostOrchestratorClient.buildListCvdsRequest; 28 import static com.android.tradefed.util.avd.HostOrchestratorClient.buildPowerwashRequest; 29 import static com.android.tradefed.util.avd.HostOrchestratorClient.buildRemoveInstanceRequest; 30 import static com.android.tradefed.util.avd.HostOrchestratorClient.saveToFile; 31 import static com.android.tradefed.util.avd.HostOrchestratorClient.sendRequest; 32 33 import com.android.ddmlib.Log.LogLevel; 34 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 35 import com.android.tradefed.log.LogUtil.CLog; 36 import com.android.tradefed.util.CommandResult; 37 import com.android.tradefed.util.CommandStatus; 38 import com.android.tradefed.util.FileUtil; 39 import com.android.tradefed.util.IRunUtil; 40 import com.android.tradefed.util.RunUtil; 41 import com.android.tradefed.util.ZipUtil2; 42 import com.android.tradefed.util.avd.OxygenClient.LHPTunnelMode; 43 44 import com.google.common.annotations.VisibleForTesting; 45 46 import org.json.JSONException; 47 import org.json.JSONObject; 48 49 import java.io.File; 50 import java.io.FileOutputStream; 51 import java.io.IOException; 52 import java.net.URI; 53 import java.net.http.HttpRequest; 54 import java.nio.file.Files; 55 import java.nio.file.Paths; 56 import java.util.ArrayList; 57 import java.util.List; 58 import java.util.Map; 59 import java.util.concurrent.TimeoutException; 60 61 /** Utility to execute commands via Host Orchestrator on remote instances. */ 62 public class HostOrchestratorUtil { 63 public static final String URL_HOST_KERNEL_LOG = "_journal/entries?_TRANSPORT=kernel"; 64 public static final String URL_HO_LOG = 65 "_journal/entries?_SYSTEMD_UNIT=cuttlefish-host_orchestrator.service"; 66 public static final String URL_OXYGEN_CONTAINER_LOG = "_journal/entries?CONTAINER_NAME=oxygen"; 67 private static final long CMD_TIMEOUT_MS = 5 * 6 * 1000 * 10; // 5 min 68 private static final long WAIT_FOR_OPERATION_MS = 5 * 1000; // 5 sec 69 private static final long WAIT_FOR_OPERATION_TIMEOUT_MS = 5 * 6 * 1000 * 10; // 5 min 70 private static final String CVD_HOST_LOGZ = "cvd_hostlog_zip"; 71 private static final String URL_CVD_BUGREPORTS = "cvdbugreports/%s"; 72 private static final String UNSUPPORTED_API_RESPONSE = "404 page not found"; 73 74 private File mTunnelLog; 75 private FileOutputStream mTunnelLogStream; 76 private boolean mUseOxygenation = false; 77 private boolean mUseCvdOxygen = false; 78 79 // User name and key file to ssh to the host VM 80 private File mSshPrivateKeyPath; 81 private String mInstanceUser; 82 83 // Oxygen instance name, host name, target region, accounting user, device ID, and other oxygen 84 // arguments. 85 private String mInstanceName; 86 private String mHost; 87 private String mOxygenationDeviceId; 88 private String mTargetRegion; 89 private String mAccountingUser; 90 private Map<String, String> mExtraOxygenArgs; 91 private OxygenClient mOxygenClient; 92 private IHoHttpClient mHttpClient; 93 private String mHOPortNumber = "2080"; 94 private Process mHOTunnel; 95 HostOrchestratorUtil( boolean useOxygenation, Map<String, String> extraOxygenArgs, String instanceName, String host, String oxygenationDeviceId, String targetRegion, String accountingUser, OxygenClient oxygenClient)96 public HostOrchestratorUtil( 97 boolean useOxygenation, 98 Map<String, String> extraOxygenArgs, 99 String instanceName, 100 String host, 101 String oxygenationDeviceId, 102 String targetRegion, 103 String accountingUser, 104 OxygenClient oxygenClient) { 105 this( 106 useOxygenation, 107 extraOxygenArgs, 108 instanceName, 109 host, 110 oxygenationDeviceId, 111 targetRegion, 112 accountingUser, 113 oxygenClient, 114 new HoHttpClient()); 115 } 116 HostOrchestratorUtil( boolean useOxygenation, Map<String, String> extraOxygenArgs, String instanceName, String host, String oxygenationDeviceId, String targetRegion, String accountingUser, OxygenClient oxygenClient, IHoHttpClient httpClient)117 public HostOrchestratorUtil( 118 boolean useOxygenation, 119 Map<String, String> extraOxygenArgs, 120 String instanceName, 121 String host, 122 String oxygenationDeviceId, 123 String targetRegion, 124 String accountingUser, 125 OxygenClient oxygenClient, 126 IHoHttpClient httpClient) { 127 mUseOxygenation = useOxygenation; 128 mExtraOxygenArgs = extraOxygenArgs; 129 mInstanceName = instanceName; 130 mHost = host; 131 mOxygenationDeviceId = oxygenationDeviceId; 132 mTargetRegion = targetRegion; 133 mAccountingUser = accountingUser; 134 mOxygenClient = oxygenClient; 135 mHttpClient = httpClient; 136 if (mUseOxygenation) { 137 mHOTunnel = createHostOrchestratorTunnel(); 138 } 139 } 140 141 /** 142 * Download log files. 143 * 144 * @param logName the log name to use when reporting to the {@link ITestLogger} 145 * @param urlPath url path indicating the log to download. 146 */ downloadLogFile(String logName, String urlPath)147 public File downloadLogFile(String logName, String urlPath) { 148 File tempFile = null; 149 try { 150 tempFile = Files.createTempFile(logName, ".txt").toFile(); 151 if (mUseOxygenation) { 152 if (mHOTunnel == null || !mHOTunnel.isAlive()) { 153 CLog.e("Failed portforwarding Host Orchestrator tunnel."); 154 FileUtil.deleteFile(tempFile); 155 return null; 156 } 157 } 158 String baseUrl = getHOBaseUrl(mHOPortNumber); 159 HttpRequest request = 160 HttpRequest.newBuilder().uri(URI.create(baseUrl + "/" + urlPath)).build(); 161 saveToFile(mHttpClient, request, Paths.get(tempFile.getAbsolutePath())); 162 return tempFile; 163 } catch (IOException | InterruptedException | ErrorResponseException e) { 164 CLog.e("Failed downloading logs with url path %s: %s", urlPath, e); 165 FileUtil.deleteFile(tempFile); 166 return null; 167 } 168 } 169 170 /** Pull CF host logs via Host Orchestrator. */ pullCvdHostLogs()171 public File pullCvdHostLogs() { 172 // Basically, the rough processes to pull CF host logs are 173 // 1. Portforward CURL tunnel. 174 // 2. Run /cvds API and parse the json to get ${GROUP_NAME}. 175 // 3. Run /cvds/${GROUP_NAME}/:bugreport to get ${OPERATION_ID} 176 // 4. Periodically run /operations/${OPERATION_ID}, parse the json util get "done":true. 177 // 5. Run /operations/${OPERATION_ID}/result to get the ${UUID}. 178 // 6. Run /cvdbugreports/${UUID} to download the artifact. 179 File cvdLogsDir = null; 180 File cvdLogsZip = null; 181 try { 182 cvdLogsZip = Files.createTempFile(CVD_HOST_LOGZ, ".zip").toFile(); 183 if (mUseOxygenation) { 184 if (mHOTunnel == null || !mHOTunnel.isAlive()) { 185 CLog.e("Failed portforwarding Host Orchestrator CURL tunnel."); 186 return null; 187 } 188 } 189 ListCvdsResponse listCvdsRes = listCvds(); 190 if (listCvdsRes.cvds.size() == 0) { 191 CLog.e("No cvd found."); 192 return null; 193 } 194 String cvdGroup = listCvdsRes.cvds.get(0).group; 195 String baseUrl = getHOBaseUrl(mHOPortNumber); 196 HttpRequest httpRequest = buildCreateBugreportRequest(baseUrl, cvdGroup); 197 Operation operation = sendRequest(mHttpClient, httpRequest, Operation.class); 198 waitForOperation(mHttpClient, baseUrl, operation.name, WAIT_FOR_OPERATION_TIMEOUT_MS); 199 httpRequest = buildGetOperationResultRequest(baseUrl, operation.name); 200 String bugreportId = sendRequest(mHttpClient, httpRequest, String.class); 201 CommandResult curlRes = 202 curlCommandExecution( 203 mHOPortNumber, 204 "GET", 205 String.format(URL_CVD_BUGREPORTS, bugreportId), 206 true, 207 "--output", 208 cvdLogsZip.getAbsolutePath()); 209 if (!CommandStatus.SUCCESS.equals(curlRes.getStatus())) { 210 CLog.e( 211 "Failed downloading cvd host logs via Host Orchestrator: %s", 212 curlRes.getStdout()); 213 return null; 214 } 215 cvdLogsDir = ZipUtil2.extractZipToTemp(cvdLogsZip, "cvd_logs"); 216 } catch (IOException | InterruptedException | ErrorResponseException | TimeoutException e) { 217 CLog.e("Failed pulling cvd host logs via Host Orchestrator: %s", e); 218 } finally { 219 cvdLogsZip.delete(); 220 } 221 return cvdLogsDir; 222 } 223 224 /** 225 * Get CF running status via Host Orchestrator. 226 * 227 * @param maxWaitTime The max timeout expected to getting the CF running status. 228 * @return True if device boot complete, false otherwise. 229 */ deviceBootCompleted(long maxWaitTime)230 public boolean deviceBootCompleted(long maxWaitTime) { 231 if (mUseOxygenation) { 232 if (mHOTunnel == null || !mHOTunnel.isAlive()) { 233 CLog.e("Failed portforwarding Host Orchestrator CURL tunnel."); 234 return false; 235 } 236 } 237 long maxEndTime = System.currentTimeMillis() + maxWaitTime; 238 while (System.currentTimeMillis() < maxEndTime) { 239 ListCvdsResponse listCvdsRes = null; 240 try { 241 listCvdsRes = listCvds(); 242 } catch (IOException | InterruptedException | ErrorResponseException e) { 243 CLog.e("Failed listing cvds: %s", e); 244 return false; 245 } 246 if (listCvdsRes.cvds.size() > 0 && listCvdsRes.cvds.get(0).status.equals("Running")) { 247 return true; 248 } 249 getRunUtil().sleep(WAIT_FOR_OPERATION_MS); 250 } 251 return false; 252 } 253 254 /** 255 * Performs list cvds request against HO. 256 * 257 * @return A {@link ListCvdsResponse} response of list cvds request. 258 */ listCvds()259 ListCvdsResponse listCvds() throws InterruptedException, IOException, ErrorResponseException { 260 String baseUrl = getHOBaseUrl(mHOPortNumber); 261 HttpRequest httpRequest = buildListCvdsRequest(baseUrl); 262 return sendRequest(mHttpClient, httpRequest, ListCvdsResponse.class); 263 } 264 265 /** 266 * Attempt to powerwash a GCE instance via Host Orchestrator. 267 * 268 * @return A {@link CommandResult} containing the status and logs. 269 */ powerwashGce()270 public CommandResult powerwashGce() { 271 // Basically, the rough processes to powerwash a GCE instance are 272 // 1. Portforward CURL tunnel 273 // 2. Obtain the necessary information to powerwash a GCE instance via Host Orchestrator. 274 // 3. Attempt to powerwash a GCE instance via Host Orchestrator. 275 // TODO(easoncylee): Flesh out this section when it's ready. 276 try { 277 if (mUseOxygenation) { 278 if (mHOTunnel == null || !mHOTunnel.isAlive()) { 279 CommandResult curlRes = new CommandResult(CommandStatus.EXCEPTION); 280 String msg = "Failed portforwarding Host Orchestrator tunnel."; 281 CLog.e(msg); 282 curlRes.setStderr(msg); 283 return curlRes; 284 } 285 } 286 ListCvdsResponse listCvdsRes = listCvds(); 287 if (listCvdsRes.cvds.size() == 0) { 288 CLog.e("No cvd found."); 289 return null; 290 } 291 Cvd cvd = listCvdsRes.cvds.get(0); 292 String baseUrl = getHOBaseUrl(mHOPortNumber); 293 HttpRequest httpRequest = buildPowerwashRequest(baseUrl, cvd.group, cvd.name); 294 Operation operation = sendRequest(mHttpClient, httpRequest, Operation.class); 295 waitForOperation(mHttpClient, baseUrl, operation.name, WAIT_FOR_OPERATION_TIMEOUT_MS); 296 } catch (IOException | InterruptedException | ErrorResponseException | TimeoutException e) { 297 CLog.e("Failed powerwashing gce via Host Orchestrator: %s", e); 298 return new CommandResult(CommandStatus.EXCEPTION); 299 } 300 return new CommandResult(CommandStatus.SUCCESS); 301 } 302 303 /** Remove Cuttlefish instance via Host Orchestrator. */ removeInstance()304 public CommandResult removeInstance() { 305 // Basically, the rough processes to remove an instance are 306 // 1. Portforward CURL tunnel 307 // 2. Obtain the group and instance name. 308 // 3. Attempt to remove the Instance via Host Orchestrator. 309 try { 310 if (mUseOxygenation) { 311 if (mHOTunnel == null || !mHOTunnel.isAlive()) { 312 CommandResult curlRes = new CommandResult(CommandStatus.EXCEPTION); 313 String msg = "Failed portforwarding Host Orchestrator tunnel."; 314 CLog.e(msg); 315 curlRes.setStderr(msg); 316 return curlRes; 317 } 318 } 319 ListCvdsResponse listCvdsRes = listCvds(); 320 if (listCvdsRes.cvds.size() == 0) { 321 CLog.e("No cvd found."); 322 return null; 323 } 324 Cvd cvd = listCvdsRes.cvds.get(0); 325 String baseUrl = getHOBaseUrl(mHOPortNumber); 326 HttpRequest httpRequest = buildRemoveInstanceRequest(baseUrl, cvd.group, cvd.name); 327 Operation operation = sendRequest(mHttpClient, httpRequest, Operation.class); 328 waitForOperation(mHttpClient, baseUrl, operation.name, WAIT_FOR_OPERATION_TIMEOUT_MS); 329 } catch (IOException | InterruptedException | ErrorResponseException | TimeoutException e) { 330 CLog.e("Failed removing instance via Host Orchestrator: %s", e); 331 return new CommandResult(CommandStatus.EXCEPTION); 332 } 333 return new CommandResult(CommandStatus.SUCCESS); 334 } 335 336 /** Attempt to snapshot a Cuttlefish instance via Host Orchestrator. */ snapshotGce()337 public CommandResult snapshotGce() { 338 // TODO(b/339304559): Flesh out this section when the host orchestrator is supported. 339 return new CommandResult(CommandStatus.EXCEPTION); 340 } 341 342 /** Attempt to restore snapshot of a Cuttlefish instance via Host Orchestrator. */ restoreSnapshotGce()343 public CommandResult restoreSnapshotGce() { 344 // TODO(b/339304559): Flesh out this section when the host orchestrator is supported. 345 return new CommandResult(CommandStatus.EXCEPTION); 346 } 347 348 /** Attempt to delete snapshot of a Cuttlefish instance via Host Orchestrator. */ deleteSnapshotGce(String snapshotId)349 public CommandResult deleteSnapshotGce(String snapshotId) { 350 // TODO(b/339304559): Flesh out this section when the host orchestrator is supported. 351 return new CommandResult(CommandStatus.EXCEPTION); 352 } 353 354 /** 355 * Create Host Orchestrator Tunnel with an automatically created port number. 356 * 357 * @return A {@link Process} of the Host Orchestrator connection between CuttleFish and TF. 358 */ 359 @VisibleForTesting createHostOrchestratorTunnel()360 Process createHostOrchestratorTunnel() { 361 mHOPortNumber = Integer.toString(mOxygenClient.createServerSocket()); 362 if (mTunnelLog == null || !mTunnelLog.exists()) { 363 try { 364 mTunnelLog = FileUtil.createTempFile("host-orchestrator-connection", ".txt"); 365 mTunnelLogStream = new FileOutputStream(mTunnelLog, true); 366 } catch (IOException e) { 367 FileUtil.deleteFile(mTunnelLog); 368 CLog.e(e); 369 } 370 } 371 CLog.i("Portforwarding host orchestrator for oxygenation CF."); 372 return mOxygenClient.createTunnelViaLHP( 373 LHPTunnelMode.CURL, 374 mHOPortNumber, 375 mInstanceName, 376 mHost, 377 mTargetRegion, 378 mAccountingUser, 379 mOxygenationDeviceId, 380 mExtraOxygenArgs, 381 mTunnelLogStream); 382 } 383 384 /** 385 * Execute a curl command via Host Orchestrator. 386 * 387 * @param portNumber The port number that Host Orchestrator communicates with. 388 * @param method The HTTP Request containing GET, POST, PUT, DELETE, PATCH, etc... 389 * @param api The API that Host Orchestrator supports. 390 * @param commands The command to be executed. 391 * @return A {@link CommandResult} containing the status and logs. 392 */ 393 @VisibleForTesting curlCommandExecution( String portNumber, String method, String api, boolean shouldDisplay, String... commands)394 CommandResult curlCommandExecution( 395 String portNumber, 396 String method, 397 String api, 398 boolean shouldDisplay, 399 String... commands) { 400 List<String> cmd = new ArrayList<>(); 401 cmd.add("curl"); 402 cmd.add("-0"); 403 cmd.add("-v"); 404 cmd.add("-X"); 405 cmd.add(method); 406 cmd.add(getHOBaseUrl(portNumber) + "/" + api); 407 for (String cmdOption : commands) { 408 cmd.add(cmdOption); 409 } 410 CommandResult commandRes = 411 getRunUtil().runTimedCmd(CMD_TIMEOUT_MS, null, null, cmd.toArray(new String[0])); 412 if (shouldDisplay) { 413 CLog.logAndDisplay( 414 LogLevel.INFO, 415 "Executing Host Orchestrator curl command: %s, Stdout: %s, Stderr: %s, Status:" 416 + " %s", 417 cmd, 418 commandRes.getStdout(), 419 commandRes.getStderr(), 420 commandRes.getStatus()); 421 } 422 if (commandRes.getStdout().contains(UNSUPPORTED_API_RESPONSE)) { 423 commandRes.setStatus(CommandStatus.FAILED); 424 InvocationMetricLogger.addInvocationMetrics( 425 InvocationMetricLogger.InvocationMetricKey.UNSUPPORTED_HOST_ORCHESTRATOR_API, 426 api); 427 } 428 return commandRes; 429 } 430 431 /** Return the value by parsing the simple JSON content with a given keyword. */ parseCvdContent(String content, String keyword)432 private String parseCvdContent(String content, String keyword) { 433 String output = ""; 434 try { 435 JSONObject object = new JSONObject(content); 436 output = object.get(keyword).toString(); 437 } catch (JSONException e) { 438 CLog.e(e); 439 } 440 return output; 441 } 442 443 /** 444 * Execute long-run operations via Host Orchestrator. A certain HO APIs would take longer time 445 * to complete, in order not to execute a long-run operations and wait for the output. This 446 * method calls the operation, get the operation id, periodically do quick check the operation's 447 * status util it's done, and return the result. 448 * 449 * @param portNumber The port number that Host Orchestrator communicates with. 450 * @param method The HTTP Request containing GET, POST, PUT, DELETE, PATCH, etc... 451 * @param request The HTTP request to be executed. 452 * @param maxWaitTime The max timeout expected to execute the HTTP request. 453 * @return A CommandResult containing the status and logs after running curl command. 454 */ 455 @VisibleForTesting cvdOperationExecution( IHoHttpClient client, String portNumber, String method, String request, long maxWaitTime)456 CommandResult cvdOperationExecution( 457 IHoHttpClient client, 458 String portNumber, 459 String method, 460 String request, 461 long maxWaitTime) 462 throws IOException, InterruptedException, ErrorResponseException { 463 CommandResult commandRes = curlCommandExecution(portNumber, method, request, true); 464 if (!CommandStatus.SUCCESS.equals(commandRes.getStatus())) { 465 CLog.e("Failed running %s, error: %s", request, commandRes.getStdout()); 466 return commandRes; 467 } 468 String operationId = parseCvdContent(commandRes.getStdout(), "name"); 469 long maxEndTime = System.currentTimeMillis() + maxWaitTime; 470 while (System.currentTimeMillis() < maxEndTime) { 471 HttpRequest httpRequest = 472 buildGetOperationRequest(getHOBaseUrl(portNumber), operationId); 473 Operation op = sendRequest(client, httpRequest, Operation.class); 474 if (op.done) { 475 return commandRes; 476 } 477 getRunUtil().sleep(WAIT_FOR_OPERATION_MS); 478 } 479 CLog.e("Running long operation cvd request timedout!"); 480 // Return the last command result and change the status to TIMED_OUT. 481 commandRes.setStatus(CommandStatus.TIMED_OUT); 482 InvocationMetricLogger.addInvocationMetrics( 483 InvocationMetricLogger.InvocationMetricKey.CVD_LONG_OPERATION_TIMEOUT_API, request); 484 return commandRes; 485 } 486 487 /** 488 * Wait for operation to finish or timeout. 489 * 490 * @param client http client to perm 491 * @param name Operation name. 492 * @param maxWaitTime waiting time, if reached out, an execption will be thrown. 493 */ waitForOperation( IHoHttpClient client, String baseUrl, String name, long maxWaitTimeMs)494 public void waitForOperation( 495 IHoHttpClient client, String baseUrl, String name, long maxWaitTimeMs) 496 throws IOException, InterruptedException, TimeoutException, ErrorResponseException { 497 long maxEndTime = System.currentTimeMillis() + maxWaitTimeMs; 498 while (System.currentTimeMillis() < maxEndTime) { 499 HttpRequest httpRequest = buildGetOperationRequest(baseUrl, name); 500 Operation op = sendRequest(client, httpRequest, Operation.class); 501 if (op.done) { 502 return; 503 } 504 getRunUtil().sleep(WAIT_FOR_OPERATION_MS); 505 } 506 CLog.e("Timeout waiting for operation: " + name); 507 throw new TimeoutException("Operation wait timeout, operation name: " + name); 508 } 509 510 /** Get {@link IRunUtil} to use. Exposed for unit testing. */ 511 // TODO(dshi): Restore VisibleForTesting after the unittest is moved to the same package 512 // (tradefed-avd-util) getRunUtil()513 protected IRunUtil getRunUtil() { 514 return RunUtil.getDefault(); 515 } 516 517 /** Return the unsupported api response. Exposed for unit testing. */ 518 @VisibleForTesting getUnsupportedHoResponse()519 String getUnsupportedHoResponse() { 520 return UNSUPPORTED_API_RESPONSE; 521 } 522 523 /** Return the host orchestrator tunnel log file. */ getTunnelLog()524 public File getTunnelLog() { 525 return mTunnelLog; 526 } 527 528 /** Return the host orchestrator URL. */ getHOBaseUrl(String port)529 String getHOBaseUrl(String port) { 530 String host = mUseOxygenation ? "127.0.0.1" : mHost; 531 return String.format("http://%s:%s", host, port); 532 } 533 534 /** Close the connection to the remote oxygenation device with a given {@link Process}. */ closeTunnelConnection()535 public void closeTunnelConnection() { 536 if (mUseOxygenation) { 537 mOxygenClient.closeLHPConnection(mHOTunnel); 538 } 539 } 540 } 541