1 /* 2 * Copyright (C) 2023 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.adservices.service.common.httpclient; 18 19 import static android.adservices.exceptions.RetryableAdServicesNetworkException.DEFAULT_RETRY_AFTER_VALUE; 20 21 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_HTTP_REQUEST_ERROR; 22 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_HTTP_REQUEST_RETRIABLE_ERROR; 23 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_OPENING_URL_FAILED; 24 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_TIMEOUT_READING_RESPONSE; 25 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_URI_IS_MALFORMED; 26 import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__COMMON; 27 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_NETWORK_FAILURE; 28 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_SUCCESS; 29 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_TIMEOUT; 30 import static com.android.adservices.service.stats.AdsRelevanceStatusUtils.ENCODING_FETCH_STATUS_UNSET; 31 32 import android.adservices.exceptions.AdServicesNetworkException; 33 import android.adservices.exceptions.RetryableAdServicesNetworkException; 34 import android.annotation.NonNull; 35 import android.annotation.Nullable; 36 import android.annotation.SuppressLint; 37 import android.net.Uri; 38 39 import com.android.adservices.LogUtil; 40 import com.android.adservices.errorlogging.ErrorLogUtil; 41 import com.android.adservices.service.common.ValidatorUtil; 42 import com.android.adservices.service.common.WebAddresses; 43 import com.android.adservices.service.common.cache.CacheProviderFactory; 44 import com.android.adservices.service.common.cache.DBCacheEntry; 45 import com.android.adservices.service.common.cache.HttpCache; 46 import com.android.adservices.service.devapi.DevContext; 47 import com.android.adservices.service.exception.HttpContentSizeException; 48 import com.android.adservices.service.profiling.Tracing; 49 import com.android.adservices.service.stats.FetchProcessLogger; 50 import com.android.adservices.service.stats.FetchProcessLoggerNoLoggingImpl; 51 import com.android.internal.annotations.VisibleForTesting; 52 53 import com.google.common.base.Charsets; 54 import com.google.common.base.Preconditions; 55 import com.google.common.collect.ImmutableMap; 56 import com.google.common.collect.ImmutableSet; 57 import com.google.common.io.BaseEncoding; 58 import com.google.common.io.ByteStreams; 59 import com.google.common.util.concurrent.ClosingFuture; 60 import com.google.common.util.concurrent.ListenableFuture; 61 import com.google.common.util.concurrent.ListeningExecutorService; 62 import com.google.common.util.concurrent.MoreExecutors; 63 64 import java.io.BufferedInputStream; 65 import java.io.BufferedOutputStream; 66 import java.io.ByteArrayOutputStream; 67 import java.io.Closeable; 68 import java.io.IOException; 69 import java.io.InputStream; 70 import java.io.OutputStream; 71 import java.io.OutputStreamWriter; 72 import java.net.HttpURLConnection; 73 import java.net.MalformedURLException; 74 import java.net.SocketTimeoutException; 75 import java.net.URL; 76 import java.net.URLConnection; 77 import java.nio.charset.StandardCharsets; 78 import java.security.SecureRandom; 79 import java.security.cert.X509Certificate; 80 import java.time.Duration; 81 import java.util.ArrayList; 82 import java.util.HashMap; 83 import java.util.List; 84 import java.util.Map; 85 import java.util.Objects; 86 import java.util.concurrent.ExecutorService; 87 88 import javax.net.ssl.HttpsURLConnection; 89 import javax.net.ssl.SSLContext; 90 import javax.net.ssl.SSLSocketFactory; 91 import javax.net.ssl.TrustManager; 92 import javax.net.ssl.X509TrustManager; 93 94 /** 95 * This is an HTTPS client to be used by the PP API services. The primary uses of this client 96 * include fetching payloads from ad tech-provided URIs and reporting on generated reporting URLs 97 * through GET or POST calls. 98 */ 99 public class AdServicesHttpsClient { 100 101 public static final long DEFAULT_MAX_BYTES = 1048576; 102 public static final int DEFAULT_TIMEOUT_MS = 5000; 103 private static final int CEL_PPAPI_NAME = AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__COMMON; 104 // Setting default max content size to 1024 * 1024 which is ~ 1MB 105 private static final String CONTENT_SIZE_ERROR = "Content size exceeds limit!"; 106 private static final String RETRY_AFTER_HEADER_FIELD = "Retry-After"; 107 private final int mConnectTimeoutMs; 108 private final int mReadTimeoutMs; 109 private final long mMaxBytes; 110 private final ListeningExecutorService mExecutorService; 111 private final UriConverter mUriConverter; 112 private final HttpCache mCache; 113 114 /** 115 * Create an HTTPS client with the input {@link ExecutorService} and initial connect and read 116 * timeouts (in milliseconds). Using this constructor does not provide any caching. 117 * 118 * @param executorService an {@link ExecutorService} that allows connection and fetching to be 119 * executed outside the main calling thread 120 * @param connectTimeoutMs the timeout, in milliseconds, for opening an initial link with to a 121 * target resource using this client. If set to 0, this timeout is interpreted as infinite 122 * (see {@link URLConnection#setConnectTimeout(int)}). 123 * @param readTimeoutMs the timeout, in milliseconds, for reading a response from a target 124 * address using this client. If set to 0, this timeout is interpreted as infinite (see 125 * {@link URLConnection#setReadTimeout(int)}). 126 * @param maxBytes The maximum size of an HTTPS response in bytes. 127 */ AdServicesHttpsClient( ExecutorService executorService, int connectTimeoutMs, int readTimeoutMs, long maxBytes)128 public AdServicesHttpsClient( 129 ExecutorService executorService, 130 int connectTimeoutMs, 131 int readTimeoutMs, 132 long maxBytes) { 133 this( 134 executorService, 135 connectTimeoutMs, 136 readTimeoutMs, 137 maxBytes, 138 new UriConverter(), 139 CacheProviderFactory.createNoOpCache()); 140 } 141 142 /** 143 * Create an HTTPS client with the input {@link ExecutorService} and default initial connect and 144 * read timeouts. This will also contain the default size of an HTTPS response, 1 MB. 145 * 146 * @param executorService an {@link ExecutorService} that allows connection and fetching to be 147 * executed outside the main calling thread 148 * @param cache A {@link HttpCache} that caches requests and response based on the use case 149 */ AdServicesHttpsClient( @onNull ExecutorService executorService, @NonNull HttpCache cache)150 public AdServicesHttpsClient( 151 @NonNull ExecutorService executorService, @NonNull HttpCache cache) { 152 this( 153 executorService, 154 DEFAULT_TIMEOUT_MS, 155 DEFAULT_TIMEOUT_MS, 156 DEFAULT_MAX_BYTES, 157 new UriConverter(), 158 cache); 159 } 160 161 @VisibleForTesting AdServicesHttpsClient( ExecutorService executorService, int connectTimeoutMs, int readTimeoutMs, long maxBytes, UriConverter uriConverter, @NonNull HttpCache cache)162 AdServicesHttpsClient( 163 ExecutorService executorService, 164 int connectTimeoutMs, 165 int readTimeoutMs, 166 long maxBytes, 167 UriConverter uriConverter, 168 @NonNull HttpCache cache) { 169 mConnectTimeoutMs = connectTimeoutMs; 170 mReadTimeoutMs = readTimeoutMs; 171 mExecutorService = MoreExecutors.listeningDecorator(executorService); 172 mMaxBytes = maxBytes; 173 mUriConverter = uriConverter; 174 mCache = cache; 175 } 176 177 /** Opens the Url Connection */ 178 @NonNull openUrl(@onNull URL url)179 private URLConnection openUrl(@NonNull URL url) throws IOException { 180 Objects.requireNonNull(url); 181 return url.openConnection(); 182 } 183 184 @NonNull setupConnection(@onNull URL url, @NonNull DevContext devContext)185 private HttpsURLConnection setupConnection(@NonNull URL url, @NonNull DevContext devContext) 186 throws IOException { 187 Objects.requireNonNull(url); 188 Objects.requireNonNull(devContext); 189 // We validated that the URL is https in toUrl 190 HttpsURLConnection urlConnection = (HttpsURLConnection) openUrl(url); 191 urlConnection.setConnectTimeout(mConnectTimeoutMs); 192 urlConnection.setReadTimeout(mReadTimeoutMs); 193 // Setting true explicitly to follow redirects 194 Uri uri = Uri.parse(url.toString()); 195 if (WebAddresses.isLocalhost(uri) && devContext.getDeviceDevOptionsEnabled()) { 196 LogUtil.v("Using unsafe HTTPS for url %s", url.toString()); 197 urlConnection.setSSLSocketFactory(getUnsafeSslSocketFactory()); 198 } else if (WebAddresses.isLocalhost(uri)) { 199 LogUtil.v( 200 String.format( 201 "Using normal HTTPS without unsafe SSL socket factory for a localhost" 202 + " address, DevOptionsEnabled: %s, CallingPackageName: %s", 203 devContext.getDeviceDevOptionsEnabled(), 204 devContext.getCallingAppPackageName())); 205 } 206 urlConnection.setInstanceFollowRedirects(true); 207 return urlConnection; 208 } 209 210 @NonNull setupPostConnectionWithPlainText( @onNull URL url, @NonNull DevContext devContext)211 private HttpsURLConnection setupPostConnectionWithPlainText( 212 @NonNull URL url, @NonNull DevContext devContext) throws IOException { 213 Objects.requireNonNull(url); 214 Objects.requireNonNull(devContext); 215 HttpsURLConnection urlConnection = setupConnection(url, devContext); 216 urlConnection.setRequestMethod("POST"); 217 urlConnection.setRequestProperty("Content-Type", "text/plain"); 218 urlConnection.setDoOutput(true); 219 return urlConnection; 220 } 221 222 @SuppressLint({"TrustAllX509TrustManager", "CustomX509TrustManager"}) getUnsafeSslSocketFactory()223 private static SSLSocketFactory getUnsafeSslSocketFactory() { 224 try { 225 TrustManager[] bypassTrustManagers = 226 new TrustManager[] { 227 new X509TrustManager() { 228 public X509Certificate[] getAcceptedIssuers() { 229 return new X509Certificate[0]; 230 } 231 232 public void checkClientTrusted( 233 X509Certificate[] chain, String authType) {} 234 235 public void checkServerTrusted( 236 X509Certificate[] chain, String authType) {} 237 } 238 }; 239 SSLContext sslContext = SSLContext.getInstance("TLS"); 240 sslContext.init(null, bypassTrustManagers, new SecureRandom()); 241 return sslContext.getSocketFactory(); 242 } catch (Exception e) { 243 LogUtil.e(e, "getUnsafeSslSocketFactory caught exception"); 244 return null; 245 } 246 } 247 248 /** 249 * Performs a GET request on the given URI in order to fetch a payload. 250 * 251 * @param uri a {@link Uri} pointing to a target server, converted to a URL for fetching 252 * @return a string containing the fetched payload 253 */ 254 @NonNull fetchPayload( @onNull Uri uri, @NonNull DevContext devContext)255 public ListenableFuture<AdServicesHttpClientResponse> fetchPayload( 256 @NonNull Uri uri, @NonNull DevContext devContext) { 257 LogUtil.v("Fetching payload from uri: " + uri); 258 return fetchPayload( 259 AdServicesHttpClientRequest.builder() 260 .setUri(uri) 261 .setDevContext(devContext) 262 .build()); 263 } 264 265 /** 266 * Performs a GET request on the given URI in order to fetch a payload. with FetchProcessLogger 267 * logging. 268 * 269 * @param uri a {@link Uri} pointing to a target server, converted to a URL for fetching 270 * @return a string containing the fetched payload 271 */ 272 @NonNull fetchPayloadWithLogging( @onNull Uri uri, @NonNull DevContext devContext, @NonNull FetchProcessLogger fetchProcessLogger)273 public ListenableFuture<AdServicesHttpClientResponse> fetchPayloadWithLogging( 274 @NonNull Uri uri, 275 @NonNull DevContext devContext, 276 @NonNull FetchProcessLogger fetchProcessLogger) { 277 return fetchPayloadWithLogging( 278 AdServicesHttpClientRequest.builder().setUri(uri).setDevContext(devContext).build(), 279 fetchProcessLogger); 280 } 281 282 /** 283 * Performs a GET request on the given URI in order to fetch a payload. 284 * 285 * @param uri a {@link Uri} pointing to a target server, converted to a URL for fetching 286 * @param headers keys of the headers we want in the response 287 * @return a string containing the fetched payload 288 */ 289 @NonNull fetchPayload( @onNull Uri uri, @NonNull ImmutableSet<String> headers, @NonNull DevContext devContext)290 public ListenableFuture<AdServicesHttpClientResponse> fetchPayload( 291 @NonNull Uri uri, 292 @NonNull ImmutableSet<String> headers, 293 @NonNull DevContext devContext) { 294 LogUtil.v("Fetching payload from uri: " + uri + " with headers: " + headers.toString()); 295 return fetchPayload( 296 AdServicesHttpClientRequest.builder() 297 .setUri(uri) 298 .setResponseHeaderKeys(headers) 299 .setDevContext(devContext) 300 .build()); 301 } 302 303 /** 304 * Performs a GET request on the given URI in order to fetch a payload. 305 * 306 * @param request of type {@link AdServicesHttpClientRequest} 307 * @return a string containing the fetched payload 308 */ 309 @NonNull fetchPayload( @onNull AdServicesHttpClientRequest request)310 public ListenableFuture<AdServicesHttpClientResponse> fetchPayload( 311 @NonNull AdServicesHttpClientRequest request) { 312 return fetchPayloadWithLogging(request, new FetchProcessLoggerNoLoggingImpl()); 313 } 314 315 /** 316 * Performs a GET request on the given URI in order to fetch a payload with EncodingFetchStats 317 * logging. 318 * 319 * @param request of type {@link AdServicesHttpClientRequest} 320 * @param fetchProcessLogger of {@link FetchProcessLogger} 321 * @return a string containing the fetched payload 322 */ 323 @NonNull fetchPayloadWithLogging( @onNull AdServicesHttpClientRequest request, @NonNull FetchProcessLogger fetchProcessLogger)324 public ListenableFuture<AdServicesHttpClientResponse> fetchPayloadWithLogging( 325 @NonNull AdServicesHttpClientRequest request, 326 @NonNull FetchProcessLogger fetchProcessLogger) { 327 Objects.requireNonNull(request.getUri()); 328 329 StringBuilder logBuilder = 330 new StringBuilder( 331 "Fetching payload for request: uri: " 332 + request.getUri() 333 + " use cache: " 334 + request.getUseCache() 335 + " dev context: " 336 + request.getDevContext().getDeviceDevOptionsEnabled()); 337 if (request.getRequestProperties() != null) { 338 logBuilder 339 .append(" request properties: ") 340 .append(request.getRequestProperties().toString()); 341 } 342 if (request.getResponseHeaderKeys() != null) { 343 logBuilder 344 .append(" response headers keys to be read in response: ") 345 .append(request.getResponseHeaderKeys().toString()); 346 } 347 348 LogUtil.v(logBuilder.toString()); 349 return ClosingFuture.from( 350 mExecutorService.submit(() -> mUriConverter.toUrl(request.getUri()))) 351 .transformAsync( 352 (closer, url) -> 353 ClosingFuture.from( 354 mExecutorService.submit( 355 () -> 356 doFetchPayload( 357 url, 358 closer, 359 request, 360 fetchProcessLogger))), 361 mExecutorService) 362 .finishToFuture(); 363 } 364 doFetchPayload( @onNull URL url, @NonNull ClosingFuture.DeferredCloser closer, AdServicesHttpClientRequest request, FetchProcessLogger fetchProcessLogger)365 private AdServicesHttpClientResponse doFetchPayload( 366 @NonNull URL url, 367 @NonNull ClosingFuture.DeferredCloser closer, 368 AdServicesHttpClientRequest request, 369 FetchProcessLogger fetchProcessLogger) 370 throws IOException, AdServicesNetworkException { 371 int jsFetchStatusCode = ENCODING_FETCH_STATUS_UNSET; 372 int traceCookie = Tracing.beginAsyncSection(Tracing.FETCH_PAYLOAD); 373 LogUtil.v("Downloading payload from: \"%s\"", url.toString()); 374 if (request.getUseCache()) { 375 AdServicesHttpClientResponse cachedResponse = getResultsFromCache(url); 376 if (cachedResponse != null) { 377 jsFetchStatusCode = ENCODING_FETCH_STATUS_SUCCESS; 378 fetchProcessLogger.logEncodingJsFetchStats(jsFetchStatusCode); 379 return cachedResponse; 380 } 381 LogUtil.v("Cache miss for url: %s", url.toString()); 382 } 383 int httpTraceCookie = Tracing.beginAsyncSection(Tracing.HTTP_REQUEST); 384 HttpsURLConnection urlConnection; 385 try { 386 urlConnection = setupConnection(url, request.getDevContext()); 387 } catch (IOException e) { 388 LogUtil.d(e, "Failed to open URL"); 389 jsFetchStatusCode = ENCODING_FETCH_STATUS_NETWORK_FAILURE; 390 fetchProcessLogger.logEncodingJsFetchStats(jsFetchStatusCode); 391 throw new IllegalArgumentException("Failed to open URL!"); 392 } 393 394 InputStream inputStream = null; 395 try { 396 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 397 // necessary need to be separated 398 for (Map.Entry<String, String> entry : request.getRequestProperties().entrySet()) { 399 urlConnection.setRequestProperty(entry.getKey(), entry.getValue()); 400 } 401 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 402 Map<String, List<String>> requestPropertiesMap = urlConnection.getRequestProperties(); 403 fetchProcessLogger.startDownloadScriptTimestamp(); 404 fetchProcessLogger.startNetworkCallTimestamp(); 405 int responseCode = urlConnection.getResponseCode(); 406 fetchProcessLogger.logServerAuctionKeyFetchCalledStatsFromNetwork(responseCode); 407 fetchProcessLogger.endDownloadScriptTimestamp(responseCode); 408 LogUtil.v("Received %s response status code.", responseCode); 409 if (isSuccessfulResponse(responseCode)) { 410 inputStream = new BufferedInputStream(urlConnection.getInputStream()); 411 String responseBody = 412 fromInputStream(inputStream, urlConnection.getContentLengthLong()); 413 Map<String, List<String>> responseHeadersMap = 414 pickRequiredHeaderFields( 415 urlConnection.getHeaderFields(), request.getResponseHeaderKeys()); 416 if (request.getUseCache()) { 417 LogUtil.v("Putting data in cache for url: %s", url); 418 mCache.put(url, responseBody, requestPropertiesMap, responseHeadersMap); 419 } 420 AdServicesHttpClientResponse response = 421 AdServicesHttpClientResponse.builder() 422 .setResponseBody(responseBody) 423 .setResponseHeaders( 424 ImmutableMap.<String, List<String>>builder() 425 .putAll(responseHeadersMap.entrySet()) 426 .build()) 427 .build(); 428 jsFetchStatusCode = ENCODING_FETCH_STATUS_SUCCESS; 429 return response; 430 } else { 431 jsFetchStatusCode = ENCODING_FETCH_STATUS_NETWORK_FAILURE; 432 throwError(urlConnection, responseCode); 433 return null; 434 } 435 } catch (SocketTimeoutException e) { 436 jsFetchStatusCode = ENCODING_FETCH_STATUS_TIMEOUT; 437 throw new IOException("Connection timed out while reading response!", e); 438 } finally { 439 fetchProcessLogger.logEncodingJsFetchStats(jsFetchStatusCode); 440 maybeDisconnect(urlConnection); 441 maybeClose(inputStream); 442 Tracing.endAsyncSection(Tracing.HTTP_REQUEST, httpTraceCookie); 443 Tracing.endAsyncSection(Tracing.FETCH_PAYLOAD, traceCookie); 444 } 445 } 446 447 @VisibleForTesting pickRequiredHeaderFields( Map<String, List<String>> allHeaderFields, ImmutableSet<String> requiredHeaderKeys)448 Map<String, List<String>> pickRequiredHeaderFields( 449 Map<String, List<String>> allHeaderFields, ImmutableSet<String> requiredHeaderKeys) { 450 HashMap<String, List<String>> result = new HashMap<>(); 451 // Using lower case matching as headers are case-insensitive per 452 // https://datatracker.ietf.org/doc/html/rfc2616#section-4.2 453 Map<String, List<String>> lowerCaseHeaders = convertKeysToLowerCase(allHeaderFields); 454 for (String headerKey : requiredHeaderKeys) { 455 List<String> headerValues = 456 new ArrayList<>(getHeaderValues(lowerCaseHeaders, headerKey)); 457 if (!headerValues.isEmpty()) { 458 LogUtil.v( 459 String.format( 460 "Found header: %s in response headers with value as %s", 461 headerKey, String.join(", ", headerValues))); 462 result.put(headerKey, headerValues); 463 } 464 } 465 LogUtil.v("requiredHeaderFields: " + result); 466 return result; 467 } 468 getHeaderValues( Map<String, List<String>> allHeaderFields, String requiredField)469 private List<String> getHeaderValues( 470 Map<String, List<String>> allHeaderFields, String requiredField) { 471 if (Objects.isNull(requiredField)) { 472 return List.of(); 473 } 474 List<String> result = allHeaderFields.get(requiredField.toLowerCase()); 475 return Objects.nonNull(result) ? result : List.of(); 476 } 477 convertKeysToLowerCase( Map<String, List<String>> inputMap)478 public static Map<String, List<String>> convertKeysToLowerCase( 479 Map<String, List<String>> inputMap) { 480 if (inputMap == null) { 481 return null; 482 } 483 Map<String, List<String>> resultMap = new HashMap<>(); 484 inputMap.forEach( 485 (key, value) -> 486 resultMap.put(Objects.nonNull(key) ? key.toLowerCase() : null, value)); 487 488 return resultMap; 489 } 490 getResultsFromCache(URL url)491 private AdServicesHttpClientResponse getResultsFromCache(URL url) { 492 DBCacheEntry cachedEntry = mCache.get(url); 493 if (cachedEntry != null) { 494 LogUtil.v("Cache hit for url: %s", url.toString()); 495 return AdServicesHttpClientResponse.builder() 496 .setResponseBody(cachedEntry.getResponseBody()) 497 .setResponseHeaders( 498 ImmutableMap.<String, List<String>>builder() 499 .putAll(cachedEntry.getResponseHeaders().entrySet()) 500 .build()) 501 .build(); 502 } 503 return null; 504 } 505 506 /** 507 * Performs a GET request on a Uri without reading the response. 508 * 509 * @param uri The URI to perform the GET request on. 510 */ getAndReadNothing( @onNull Uri uri, @NonNull DevContext devContext)511 public ListenableFuture<Void> getAndReadNothing( 512 @NonNull Uri uri, @NonNull DevContext devContext) { 513 Objects.requireNonNull(uri); 514 515 return ClosingFuture.from(mExecutorService.submit(() -> mUriConverter.toUrl(uri))) 516 .transformAsync( 517 (closer, url) -> 518 ClosingFuture.from( 519 mExecutorService.submit( 520 () -> 521 doGetAndReadNothing( 522 url, closer, devContext))), 523 mExecutorService) 524 .finishToFuture(); 525 } 526 doGetAndReadNothing( @onNull URL url, @NonNull ClosingFuture.DeferredCloser closer, @NonNull DevContext devContext)527 private Void doGetAndReadNothing( 528 @NonNull URL url, 529 @NonNull ClosingFuture.DeferredCloser closer, 530 @NonNull DevContext devContext) 531 throws IOException, AdServicesNetworkException { 532 LogUtil.v( 533 "doGetAndReadNothing to: \"%s\", dev context: %s", 534 url.toString(), devContext.getDeviceDevOptionsEnabled()); 535 HttpsURLConnection urlConnection; 536 537 try { 538 urlConnection = setupConnection(url, devContext); 539 } catch (IOException e) { 540 LogUtil.d(e, "Failed to open URL"); 541 ErrorLogUtil.e( 542 e, 543 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_OPENING_URL_FAILED, 544 CEL_PPAPI_NAME); 545 throw new IllegalArgumentException("Failed to open URL!"); 546 } 547 548 try { 549 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 550 // necessary need to be separated 551 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 552 int responseCode = urlConnection.getResponseCode(); 553 if (isSuccessfulResponse(responseCode)) { 554 LogUtil.d("GET request succeeded for URL: " + url); 555 } else { 556 LogUtil.d("GET request failed for URL: " + url); 557 throwError(urlConnection, responseCode); 558 } 559 return null; 560 } catch (SocketTimeoutException e) { 561 ErrorLogUtil.e( 562 e, 563 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_TIMEOUT_READING_RESPONSE, 564 CEL_PPAPI_NAME); 565 throw new IOException("Connection timed out while reading response!", e); 566 } finally { 567 maybeDisconnect(urlConnection); 568 } 569 } 570 571 /** 572 * Performs a POST request on a Uri and attaches {@code String} to the request 573 * 574 * @param uri to do the POST request on 575 * @param requestBody Attached to the POST request. 576 */ postPlainText( @onNull Uri uri, @NonNull String requestBody, @NonNull DevContext devContext)577 public ListenableFuture<Void> postPlainText( 578 @NonNull Uri uri, @NonNull String requestBody, @NonNull DevContext devContext) { 579 Objects.requireNonNull(uri); 580 Objects.requireNonNull(requestBody); 581 582 return ClosingFuture.from(mExecutorService.submit(() -> mUriConverter.toUrl(uri))) 583 .transformAsync( 584 (closer, url) -> 585 ClosingFuture.from( 586 mExecutorService.submit( 587 () -> 588 doPostPlainText( 589 url, 590 requestBody, 591 closer, 592 devContext))), 593 mExecutorService) 594 .finishToFuture(); 595 } 596 doPostPlainText( URL url, String data, ClosingFuture.DeferredCloser closer, DevContext devContext)597 private Void doPostPlainText( 598 URL url, String data, ClosingFuture.DeferredCloser closer, DevContext devContext) 599 throws IOException, AdServicesNetworkException { 600 LogUtil.v("Reporting to: \"%s\"", url.toString()); 601 HttpsURLConnection urlConnection; 602 603 try { 604 urlConnection = setupPostConnectionWithPlainText(url, devContext); 605 } catch (IOException e) { 606 LogUtil.d(e, "Failed to open URL"); 607 ErrorLogUtil.e( 608 e, 609 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_OPENING_URL_FAILED, 610 CEL_PPAPI_NAME); 611 throw new IllegalArgumentException("Failed to open URL!"); 612 } 613 614 try { 615 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 616 // necessary need to be separated 617 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 618 619 OutputStream os = urlConnection.getOutputStream(); 620 OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8); 621 osw.write(data); 622 osw.flush(); 623 osw.close(); 624 625 int responseCode = urlConnection.getResponseCode(); 626 if (isSuccessfulResponse(responseCode)) { 627 LogUtil.d("POST request succeeded for URL: " + url); 628 } else { 629 LogUtil.d("POST request failed for URL: " + url); 630 throwError(urlConnection, responseCode); 631 } 632 return null; 633 } catch (SocketTimeoutException e) { 634 ErrorLogUtil.e( 635 e, 636 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_TIMEOUT_READING_RESPONSE, 637 CEL_PPAPI_NAME); 638 throw new IOException("Connection timed out while reading response!", e); 639 } finally { 640 maybeDisconnect(urlConnection); 641 } 642 } 643 644 /** 645 * Performs an HTTP request according to the request object and returns the response in byte 646 * array. 647 */ performRequestGetResponseInBase64String( @onNull AdServicesHttpClientRequest request)648 public ListenableFuture<AdServicesHttpClientResponse> performRequestGetResponseInBase64String( 649 @NonNull AdServicesHttpClientRequest request) { 650 return performRequestGetResponseInBase64StringWithLogging( 651 request, new FetchProcessLoggerNoLoggingImpl()); 652 } 653 654 /** 655 * Performs an HTTP request according to the request object and returns the response in byte 656 * array. 657 */ 658 public ListenableFuture<AdServicesHttpClientResponse> performRequestGetResponseInBase64StringWithLogging( @onNull AdServicesHttpClientRequest request, @NonNull FetchProcessLogger fetchProcessLogger)659 performRequestGetResponseInBase64StringWithLogging( 660 @NonNull AdServicesHttpClientRequest request, 661 @NonNull FetchProcessLogger fetchProcessLogger) { 662 Objects.requireNonNull(request.getUri()); 663 return ClosingFuture.from( 664 mExecutorService.submit(() -> mUriConverter.toUrl(request.getUri()))) 665 .transformAsync( 666 (closer, url) -> 667 ClosingFuture.from( 668 mExecutorService.submit( 669 () -> 670 doPerformRequestAndGetResponse( 671 url, 672 closer, 673 request, 674 ResponseBodyType 675 .BASE64_ENCODED_STRING, 676 fetchProcessLogger))), 677 mExecutorService) 678 .finishToFuture(); 679 } 680 681 /** 682 * Performs an HTTP request according to the request object and returns the response in plain 683 * String 684 */ performRequestGetResponseInPlainString( @onNull AdServicesHttpClientRequest request)685 public ListenableFuture<AdServicesHttpClientResponse> performRequestGetResponseInPlainString( 686 @NonNull AdServicesHttpClientRequest request) { 687 Objects.requireNonNull(request.getUri()); 688 LogUtil.d("Making request expecting a response in plain string"); 689 FetchProcessLogger fetchProcessLogger = new FetchProcessLoggerNoLoggingImpl(); 690 return ClosingFuture.from( 691 mExecutorService.submit(() -> mUriConverter.toUrl(request.getUri()))) 692 .transformAsync( 693 (closer, url) -> 694 ClosingFuture.from( 695 mExecutorService.submit( 696 () -> 697 doPerformRequestAndGetResponse( 698 url, 699 closer, 700 request, 701 ResponseBodyType.PLAIN_TEXT_STRING, 702 fetchProcessLogger))), 703 mExecutorService) 704 .finishToFuture(); 705 } 706 doPerformRequestAndGetResponse( @onNull URL url, @NonNull ClosingFuture.DeferredCloser closer, AdServicesHttpClientRequest request, ResponseBodyType responseType, @NonNull FetchProcessLogger fetchProcessLogger)707 private AdServicesHttpClientResponse doPerformRequestAndGetResponse( 708 @NonNull URL url, 709 @NonNull ClosingFuture.DeferredCloser closer, 710 AdServicesHttpClientRequest request, 711 ResponseBodyType responseType, 712 @NonNull FetchProcessLogger fetchProcessLogger) 713 throws IOException, AdServicesNetworkException { 714 HttpsURLConnection urlConnection; 715 try { 716 urlConnection = setupConnection(url, request.getDevContext()); 717 urlConnection.setRequestMethod(request.getHttpMethodType().name()); 718 } catch (IOException e) { 719 LogUtil.e(e, "Failed to open URL"); 720 ErrorLogUtil.e( 721 e, 722 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_OPENING_URL_FAILED, 723 CEL_PPAPI_NAME); 724 throw new IllegalArgumentException("Failed to open URL!"); 725 } 726 727 InputStream inputStream = null; 728 try { 729 // TODO(b/237342352): Both connect and read timeouts are kludged in this method and if 730 // necessary need to be separated 731 for (Map.Entry<String, String> entry : request.getRequestProperties().entrySet()) { 732 urlConnection.setRequestProperty(entry.getKey(), entry.getValue()); 733 } 734 735 if (request.getHttpMethodType() == AdServicesHttpUtil.HttpMethodType.POST 736 && request.getBodyInBytes() != null 737 && request.getBodyInBytes().length > 0) { 738 urlConnection.setDoOutput(true); 739 try (BufferedOutputStream out = 740 new BufferedOutputStream(urlConnection.getOutputStream())) { 741 out.write(request.getBodyInBytes()); 742 } 743 } 744 745 closer.eventuallyClose(new CloseableConnectionWrapper(urlConnection), mExecutorService); 746 fetchProcessLogger.startNetworkCallTimestamp(); 747 int responseCode = urlConnection.getResponseCode(); 748 fetchProcessLogger.logServerAuctionKeyFetchCalledStatsFromNetwork(responseCode); 749 LogUtil.v("Received %s response status code.", responseCode); 750 751 if (isSuccessfulResponse(responseCode)) { 752 LogUtil.d(" request succeeded for URL: " + url); 753 Map<String, List<String>> responseHeadersMap = 754 pickRequiredHeaderFields( 755 urlConnection.getHeaderFields(), request.getResponseHeaderKeys()); 756 inputStream = new BufferedInputStream(urlConnection.getInputStream()); 757 String responseBody; 758 if (responseType == ResponseBodyType.BASE64_ENCODED_STRING) { 759 responseBody = 760 BaseEncoding.base64() 761 .encode( 762 getByteArray( 763 inputStream, 764 urlConnection.getContentLengthLong())); 765 } else { 766 responseBody = 767 fromInputStream(inputStream, urlConnection.getContentLengthLong()); 768 } 769 return AdServicesHttpClientResponse.builder() 770 .setResponseBody(responseBody) 771 .setResponseHeaders( 772 ImmutableMap.<String, List<String>>builder() 773 .putAll(responseHeadersMap.entrySet()) 774 .build()) 775 .build(); 776 } else { 777 LogUtil.d(" request failed for URL: " + url); 778 throwError(urlConnection, responseCode); 779 return null; 780 } 781 } catch (SocketTimeoutException e) { 782 throw new IOException("Connection timed out while reading response!", e); 783 } finally { 784 maybeDisconnect(urlConnection); 785 maybeClose(inputStream); 786 } 787 } 788 getByteArray(@ullable InputStream in, long contentLength)789 private byte[] getByteArray(@Nullable InputStream in, long contentLength) throws IOException { 790 if (contentLength == 0) { 791 return new byte[0]; 792 } 793 try { 794 byte[] buffer = new byte[1024]; 795 ByteArrayOutputStream out = new ByteArrayOutputStream(); 796 int bytesRead; 797 while ((bytesRead = in.read(buffer)) != -1) { 798 out.write(buffer, 0, bytesRead); 799 } 800 return out.toByteArray(); 801 } finally { 802 in.close(); 803 } 804 } 805 throwError(final HttpsURLConnection urlConnection, int responseCode)806 private void throwError(final HttpsURLConnection urlConnection, int responseCode) 807 throws AdServicesNetworkException { 808 LogUtil.v("Error occurred while executing HTTP request."); 809 810 // Default values for AdServiceNetworkException fields. 811 @AdServicesNetworkException.ErrorCode 812 int errorCode = AdServicesNetworkException.ERROR_OTHER; 813 Duration retryAfterDuration = RetryableAdServicesNetworkException.UNSET_RETRY_AFTER_VALUE; 814 815 // Assign a relevant error code to the HTTP response code. 816 switch (responseCode / 100) { 817 case 3: 818 errorCode = AdServicesNetworkException.ERROR_REDIRECTION; 819 break; 820 case 4: 821 // If an HTTP 429 response code was received, extract the retry-after duration 822 if (responseCode == 429) { 823 errorCode = AdServicesNetworkException.ERROR_TOO_MANY_REQUESTS; 824 String headerValue = urlConnection.getHeaderField(RETRY_AFTER_HEADER_FIELD); 825 if (headerValue != null) { 826 // TODO(b/282017541): Add a maximum allowed retry-after duration. 827 retryAfterDuration = Duration.ofMillis(Long.parseLong(headerValue)); 828 } else { 829 retryAfterDuration = DEFAULT_RETRY_AFTER_VALUE; 830 } 831 } else { 832 errorCode = AdServicesNetworkException.ERROR_CLIENT; 833 } 834 break; 835 case 5: 836 errorCode = AdServicesNetworkException.ERROR_SERVER; 837 } 838 LogUtil.v("Received %s error status code.", responseCode); 839 840 // Throw the appropriate exception. 841 AdServicesNetworkException exception; 842 if (retryAfterDuration.compareTo( 843 RetryableAdServicesNetworkException.UNSET_RETRY_AFTER_VALUE) 844 <= 0) { 845 exception = new AdServicesNetworkException(errorCode); 846 ErrorLogUtil.e( 847 exception, 848 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_HTTP_REQUEST_ERROR, 849 CEL_PPAPI_NAME); 850 } else { 851 exception = new RetryableAdServicesNetworkException(errorCode, retryAfterDuration); 852 ErrorLogUtil.e( 853 exception, 854 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_HTTP_REQUEST_RETRIABLE_ERROR, 855 CEL_PPAPI_NAME); 856 } 857 LogUtil.e("Throwing %s.", exception.toString()); 858 throw exception; 859 } 860 maybeDisconnect(@ullable URLConnection urlConnection)861 private static void maybeDisconnect(@Nullable URLConnection urlConnection) { 862 if (urlConnection == null) { 863 return; 864 } 865 866 if (urlConnection instanceof HttpURLConnection) { 867 HttpURLConnection httpUrlConnection = (HttpURLConnection) urlConnection; 868 httpUrlConnection.disconnect(); 869 } else { 870 LogUtil.d("Not closing URLConnection of type %s", urlConnection.getClass()); 871 } 872 } 873 maybeClose(@ullable InputStream inputStream)874 private static void maybeClose(@Nullable InputStream inputStream) throws IOException { 875 if (inputStream == null) { 876 return; 877 } else { 878 inputStream.close(); 879 } 880 } 881 882 /** 883 * @return the connection timeout, in milliseconds, when opening an initial link to a target 884 * address with this client 885 */ getConnectTimeoutMs()886 public int getConnectTimeoutMs() { 887 return mConnectTimeoutMs; 888 } 889 890 /** 891 * @return the read timeout, in milliseconds, when reading the response from a target address 892 * with this client 893 */ getReadTimeoutMs()894 public int getReadTimeoutMs() { 895 return mReadTimeoutMs; 896 } 897 898 /** 899 * @return true if responseCode matches 2.*, i.e. 200, 204, 206 900 */ isSuccessfulResponse(int responseCode)901 public static boolean isSuccessfulResponse(int responseCode) { 902 return (responseCode / 100) == 2; 903 } 904 905 /** 906 * Reads a {@link InputStream} and returns a {@code String}. To enforce content size limits, we 907 * employ the following strategy: 1. If {@link URLConnection} cannot determine the content size, 908 * we invoke {@code manualStreamToString(InputStream)} where we manually apply the content 909 * restriction. 2. Otherwise, we invoke {@code streamToString(InputStream, long)}. 910 * 911 * @throws IOException if content size limit of is exceeded 912 */ 913 @NonNull fromInputStream(@onNull InputStream in, long size)914 private String fromInputStream(@NonNull InputStream in, long size) throws IOException { 915 Objects.requireNonNull(in); 916 if (size == 0) { 917 return ""; 918 } else if (size < 0) { 919 return manualStreamToString(in); 920 } else { 921 return streamToString(in, size); 922 } 923 } 924 925 @NonNull streamToString(@onNull InputStream in, long size)926 private String streamToString(@NonNull InputStream in, long size) throws IOException { 927 Objects.requireNonNull(in); 928 if (size > mMaxBytes) { 929 throw new HttpContentSizeException(CONTENT_SIZE_ERROR); 930 } 931 return new String(ByteStreams.toByteArray(in), Charsets.UTF_8); 932 } 933 934 @NonNull manualStreamToString(@onNull InputStream in)935 private String manualStreamToString(@NonNull InputStream in) throws IOException { 936 Objects.requireNonNull(in); 937 ByteArrayOutputStream into = new ByteArrayOutputStream(); 938 byte[] buf = new byte[1024]; 939 long total = 0; 940 for (int n; 0 < (n = in.read(buf)); ) { 941 total += n; 942 if (total <= mMaxBytes) { 943 into.write(buf, 0, n); 944 } else { 945 into.close(); 946 throw new HttpContentSizeException(CONTENT_SIZE_ERROR); 947 } 948 } 949 into.close(); 950 return into.toString("UTF-8"); 951 } 952 953 private static class CloseableConnectionWrapper implements Closeable { 954 @Nullable final HttpsURLConnection mURLConnection; 955 CloseableConnectionWrapper(HttpsURLConnection urlConnection)956 private CloseableConnectionWrapper(HttpsURLConnection urlConnection) { 957 mURLConnection = urlConnection; 958 } 959 960 @Override close()961 public void close() throws IOException { 962 maybeClose(mURLConnection.getInputStream()); 963 maybeClose(mURLConnection.getErrorStream()); 964 maybeDisconnect(mURLConnection); 965 } 966 } 967 968 /** A light-weight class to convert Uri to URL */ 969 public static final class UriConverter { 970 971 @NonNull toUrl(@onNull Uri uri)972 URL toUrl(@NonNull Uri uri) { 973 Objects.requireNonNull(uri); 974 Preconditions.checkArgument( 975 ValidatorUtil.HTTPS_SCHEME.equalsIgnoreCase(uri.getScheme()), 976 "URI \"%s\" must use HTTPS", 977 uri.toString()); 978 979 URL url; 980 try { 981 url = new URL(uri.toString()); 982 } catch (MalformedURLException e) { 983 LogUtil.d(e, "Uri is malformed! "); 984 ErrorLogUtil.e( 985 e, 986 AD_SERVICES_ERROR_REPORTED__ERROR_CODE__AD_SERVICES_HTTPS_CLIENT_URI_IS_MALFORMED, 987 AD_SERVICES_ERROR_REPORTED__PPAPI_NAME__COMMON); 988 throw new IllegalArgumentException("Uri is malformed!"); 989 } 990 return url; 991 } 992 } 993 994 /** 995 * @return the cache associated with this instance of client 996 */ getAssociatedCache()997 public HttpCache getAssociatedCache() { 998 return mCache; 999 } 1000 1001 enum ResponseBodyType { 1002 BASE64_ENCODED_STRING, 1003 PLAIN_TEXT_STRING 1004 } 1005 } 1006