xref: /sdk/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/DownloadCache.java

/*
 * Copyright (C) 2012 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.sdklib.internal.repository;

import com.android.annotations.Nullable;
import com.android.annotations.VisibleForTesting;
import com.android.annotations.VisibleForTesting.Visibility;
import com.android.prefs.AndroidLocation;
import com.android.prefs.AndroidLocation.AndroidLocationException;
import com.android.sdklib.SdkConstants;
import com.android.sdklib.internal.repository.UrlOpener.CanceledByUserException;
import com.android.util.Pair;

import org.apache.http.Header;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.message.BasicHeader;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;


/**
 * A simple cache for the XML resources handled by the SDK Manager.
 * <p/>
 * Callers should use {@link #openDirectUrl(String, ITaskMonitor)} to download "large files"
 * that should not be cached (like actual installation packages which are several MBs big)
 * and call {@link #openCachedUrl(String, ITaskMonitor)} to download small XML files.
 * <p/>
 * The cache can work in 3 different strategies (direct is a pass-through, fresh-cache is the
 * default and tries to update resources if they are older than 10 minutes by respecting
 * either ETag or Last-Modified, and finally server-cache is a strategy to always serve
 * cached entries if present.)
 */
public class DownloadCache {

    /*
     * HTTP/1.1 references:
     * - Possible headers:
     *     http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
     * - Rules about conditional requests:
     *     http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3.4
     * - Error codes:
     *     http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1
     */

    private static final boolean DEBUG = System.getenv("SDKMAN_DEBUG_CACHE") != null;

    /** Key for the Status-Code in the info properties. */
    private static final String KEY_STATUS_CODE = "Status-Code";        //$NON-NLS-1$
    /** Key for the URL in the info properties. */
    private static final String KEY_URL = "URL";                        //$NON-NLS-1$

    /** Prefix of binary files stored in the {@link SdkConstants#FD_CACHE} directory. */
    private final static String BIN_FILE_PREFIX = "sdkbin-";            //$NON-NLS-1$
    /** Prefix of meta info files stored in the {@link SdkConstants#FD_CACHE} directory. */
    private final static String INFO_FILE_PREFIX = "sdkinf-";           //$NON-NLS-1$

    /**
     * Minimum time before we consider a cached entry is potentially stale.
     * Expressed in milliseconds.
     * <p/>
     * When using the {@link Strategy#FRESH_CACHE}, the cache will not try to refresh
     * a cached file if it's has been saved more recently than this time.
     * When using the direct mode or the serve mode, the cache either doesn't serve
     * cached files or always serves caches files so this expiration delay is not used.
     * <p/>
     * Default is 10 minutes.
     * <p/>
     * TODO: change for a dynamic preference later.
     */
    private final static long MIN_TIME_EXPIRED_MS =  10*60*1000;
    /**
     * Maximum time before we consider a cache entry to be stale.
     * Expressed in milliseconds.
     * <p/>
     * When using the {@link Strategy#FRESH_CACHE}, entries that have no ETag
     * or Last-Modified will be refreshed if their file timestamp is older than
     * this value.
     * <p/>
     * Default is 4 hours.
     * <p/>
     * TODO: change for a dynamic preference later.
     */
    private final static long MAX_TIME_EXPIRED_MS = 4*60*60*1000;

    /**
     * The maximum file size we'll cache for "small" files.
     * 640KB is more than enough and is already a stretch since these are read in memory.
     * (The actual typical size of the files handled here is in the 4-64KB range.)
     */
    private final static int MAX_SMALL_FILE_SIZE = 640 * 1024;

    /**
     * HTTP Headers that are saved in an info file.
     * For HTTP/1.1 header names, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
     */
    private final static String[] INFO_HTTP_HEADERS = {
        HttpHeaders.LAST_MODIFIED,
        HttpHeaders.ETAG,
        HttpHeaders.CONTENT_LENGTH,
        HttpHeaders.DATE
    };

    private final Strategy mStrategy;
    private final File mCacheRoot;

    public enum Strategy {
        /**
         * If the files are available in the cache, serve them as-is, otherwise
         * download them and return the cached version. No expiration or refresh
         * is attempted if a file is in the cache.
         */
        SERVE_CACHE,
        /**
         * If the files are available in the cache, check if there's an update
         * (either using an e-tag check or comparing to the default time expiration).
         * If files have expired or are not in the cache then download them and return
         * the cached version.
         */
        FRESH_CACHE,
        /**
         * Disables caching. URLs are always downloaded and returned directly.
         * Downloaded streams aren't cached locally.
         */
        DIRECT
    }

    /** Creates a default instance of the URL cache */
    public DownloadCache(Strategy strategy) {
        mCacheRoot = initCacheRoot();
        mStrategy = mCacheRoot == null ? Strategy.DIRECT : strategy;
    }

    public Strategy getStrategy() {
        return mStrategy;
    }

    /**
     * Returns the directory to be used as a cache.
     * Creates it if necessary.
     * Makes it possible to disable or override the cache location in unit tests.
     *
     * @return An existing directory to use as a cache root dir,
     *   or null in case of error in which case the cache will be disabled.
     */
    @VisibleForTesting(visibility=Visibility.PRIVATE)
    protected File initCacheRoot() {
        try {
            File root = new File(AndroidLocation.getFolder());
            root = new File(root, SdkConstants.FD_CACHE);
            if (!root.exists()) {
                root.mkdirs();
            }
            return root;
        } catch (AndroidLocationException e) {
            // No root? Disable the cache.
            return null;
        }
    }

    /**
     * Does a direct download of the given URL using {@link UrlOpener}.
     * This does not check the download cache and does not attempt to cache the file.
     * Instead the HttpClient library returns a progressive download stream.
     * <p/>
     * For details on realm authentication and user/password handling,
     * check the underlying {@link UrlOpener#openUrl(String, ITaskMonitor, Header[])}
     * documentation.
     *
     * @param urlString the URL string to be opened.
     * @param monitor {@link ITaskMonitor} which is related to this URL
     *            fetching.
     * @return Returns an {@link InputStream} holding the URL content.
     * @throws IOException Exception thrown when there are problems retrieving
     *             the URL or its content.
     * @throws CanceledByUserException Exception thrown if the user cancels the
     *              authentication dialog.
     */
    public InputStream openDirectUrl(String urlString, ITaskMonitor monitor)
            throws IOException, CanceledByUserException {
        if (DEBUG) {
            System.out.println(String.format("%s : Direct download", urlString)); //$NON-NLS-1$
        }
        Pair<InputStream, HttpResponse> result =
            UrlOpener.openUrl(urlString, monitor, null /*headers*/);
        return result.getFirst();
    }

    /**
     * Downloads a small file, typically XML manifests.
     * The current {@link Strategy} governs whether the file is served as-is
     * from the cache, potentially updated first or directly downloaded.
     * <p/>
     * For large downloads (e.g. installable archives) please do not invoke the
     * cache and instead use the {@link #openDirectUrl(String, ITaskMonitor)}
     * method.
     * <p/>
     * For details on realm authentication and user/password handling,
     * check the underlying {@link UrlOpener#openUrl(String, ITaskMonitor, Header[])}
     * documentation.
     *
     * @param urlString the URL string to be opened.
     * @param monitor {@link ITaskMonitor} which is related to this URL
     *            fetching.
     * @return Returns an {@link InputStream} holding the URL content.
     * @throws IOException Exception thrown when there are problems retrieving
     *             the URL or its content.
     * @throws CanceledByUserException Exception thrown if the user cancels the
     *              authentication dialog.
     */
    public InputStream openCachedUrl(String urlString, ITaskMonitor monitor)
            throws IOException, CanceledByUserException {
        // Don't cache in direct mode. Don't try to cache non-http URLs.
        if (mStrategy == Strategy.DIRECT || !urlString.startsWith("http")) {        //$NON-NLS-1$
            return openDirectUrl(urlString, monitor);
        }

        File cached = new File(mCacheRoot, getCacheFilename(urlString));
        File info   = new File(mCacheRoot, getInfoFilename(cached.getName()));

        boolean useCached = cached.exists();

        if (useCached && mStrategy == Strategy.FRESH_CACHE) {
            // Check whether the file should be served from the cache or
            // refreshed first.

            long cacheModifiedMs = cached.lastModified(); /* last mod time in epoch/millis */
            boolean checkCache = true;

            Properties props = readInfo(info);
            if (props == null) {
                // No properties, no chocolate for you.
                useCached = false;
            } else {
                long minExpiration = System.currentTimeMillis() - MIN_TIME_EXPIRED_MS;
                checkCache = cacheModifiedMs < minExpiration;

                if (!checkCache && DEBUG) {
                    System.out.println(String.format(
                            "%s : Too fresh [%,d ms], not checking yet.",    //$NON-NLS-1$
                            urlString, cacheModifiedMs - minExpiration));
                }
            }

            if (useCached && checkCache) {
                assert props != null;

                // Right now we only support 200 codes and will requery all 404s.
                String code = props.getProperty(KEY_STATUS_CODE, "");   //$NON-NLS-1$
                useCached = Integer.toString(HttpStatus.SC_OK).equals(code);

                if (!useCached && DEBUG) {
                    System.out.println(String.format(
                            "%s : cache disabled by code %s",           //$NON-NLS-1$
                            urlString, code));
                }

                if (useCached) {
                    // Do we have a valid Content-Length? If so, it should match the file size.
                    try {
                        long length = Long.parseLong(props.getProperty(HttpHeaders.CONTENT_LENGTH,
                                                        "-1")); //$NON-NLS-1$
                        if (length >= 0) {
                            useCached = length == cached.length();

                            if (!useCached && DEBUG) {
                                System.out.println(String.format(
                                    "%s : cache disabled by length mismatch %d, expected %d", //$NON-NLS-1$
                                    urlString, length, cached.length()));
                            }
                        }
                    } catch (NumberFormatException ignore) {}
                }

                if (useCached) {
                    // Do we have an ETag and/or a Last-Modified?
                    String etag = props.getProperty(HttpHeaders.ETAG);
                    String lastMod = props.getProperty(HttpHeaders.LAST_MODIFIED);

                    if (etag != null || lastMod != null) {
                        // Details on how to use them is defined at
                        // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3.4
                        // Bottom line:
                        // - if there's an ETag, it should be used first with an
                        //   If-None-Match header. That's a strong comparison for HTTP/1.1 servers.
                        // - otherwise use a Last-Modified if an If-Modified-Since header exists.
                        // In this case, we place both and the rules indicates a spec-abiding
                        // server should strongly match ETag and weakly the Modified-Since.

                        // TODO there are some servers out there which report ETag/Last-Mod
                        // yet don't honor them when presented with a precondition. In this
                        // case we should identify it in the reply and invalidate ETag support
                        // for these servers and instead fallback on the pure-timeout case below.

                        AtomicInteger statusCode = new AtomicInteger(0);
                        InputStream is = null;
                        List<Header> headers = new ArrayList<Header>(2);

                        if (etag != null) {
                            headers.add(new BasicHeader(HttpHeaders.IF_NONE_MATCH, etag));
                        }

                        if (lastMod != null) {
                            headers.add(new BasicHeader(HttpHeaders.IF_MODIFIED_SINCE, lastMod));
                        }

                        if (!headers.isEmpty()) {
                            is = downloadAndCache(urlString, monitor, cached, info,
                                    headers.toArray(new Header[headers.size()]),
                                    statusCode);
                        }

                        if (is != null && statusCode.get() == HttpStatus.SC_OK) {
                            // The resource was modified, the server said there was something
                            // new, which has been cached. We can return that to the caller.
                            return is;
                        }

                        // If we get here, we should have is == null and code
                        // could be:
                        // - 304 for not-modified -- same resource, still available, in
                        //       which case we'll use the cached one.
                        // - 404 -- resource doesn't exist anymore in which case there's
                        //       no point in retrying.
                        // - For any other code, just retry a download.

                        if (is != null) {
                            try {
                                is.close();
                            } catch (Exception ignore) {}
                            is = null;
                        }

                        if (statusCode.get() == HttpStatus.SC_NOT_MODIFIED) {
                            // Cached file was not modified.
                            // Change its timestamp for the next MIN_TIME_EXPIRED_MS check.
                            cached.setLastModified(System.currentTimeMillis());

                            // At this point useCached==true so we'll return
                            // the cached file below.
                        } else {
                            // URL fetch returned something other than 200 or 304.
                            // For 404, we're done, no need to check the server again.
                            // For all other codes, we'll retry a download below.
                            useCached = false;
                            if (statusCode.get() == HttpStatus.SC_NOT_FOUND) {
                                return null;
                            }
                        }
                    } else {
                        // If we don't have an Etag nor Last-Modified, let's use a
                        // basic file timestamp and compare to a 1 hour threshold.

                        long maxExpiration = System.currentTimeMillis() - MAX_TIME_EXPIRED_MS;
                        useCached = cacheModifiedMs >= maxExpiration;

                        if (!useCached && DEBUG) {
                            System.out.println(String.format(
                                "[%1$s] cache disabled by timestamp %2$tD %2$tT < %3$tD %3$tT", //$NON-NLS-1$
                                urlString, cacheModifiedMs, maxExpiration));
                        }
                    }
                }
            }
        }

        if (useCached) {
            // The caller needs an InputStream that supports the reset() operation.
            // The default FileInputStream does not, so load the file into a byte
            // array and return that.
            try {
                InputStream is = readCachedFile(cached);
                if (is != null) {
                    if (DEBUG) {
                        System.out.println(String.format("%s : Use cached file", urlString)); //$NON-NLS-1$
                    }

                    return is;
                }
            } catch (IOException ignore) {}
        }

        // If we're not using the cache, try to remove the cache and download again.
        try {
            cached.delete();
            info.delete();
        } catch (SecurityException ignore) {}

        return downloadAndCache(urlString, monitor, cached, info,
                null /*headers*/, null /*statusCode*/);
    }

    // --------------

    private InputStream readCachedFile(File cached) throws IOException {
        InputStream is = null;

        int inc = 65536;
        int curr = 0;
        long len = cached.length();
        assert len < Integer.MAX_VALUE;
        if (len >= MAX_SMALL_FILE_SIZE) {
            // This is supposed to cache small files, not 2+ GB files.
            return null;
        }
        byte[] result = new byte[(int) (len > 0 ? len : inc)];

        try {
            is = new FileInputStream(cached);

            int n;
            while ((n = is.read(result, curr, result.length - curr)) != -1) {
                curr += n;
                if (curr == result.length) {
                    byte[] temp = new byte[curr + inc];
                    System.arraycopy(result, 0, temp, 0, curr);
                    result = temp;
                }
            }

            return new ByteArrayInputStream(result, 0, curr);

        } finally {
            if (is != null) {
                try {
                    is.close();
                } catch (IOException ignore) {}
            }
        }
    }

    /**
     * Download, cache and return as an in-memory byte stream.
     * The download is only done if the server returns 200/OK.
     * On success, store an info file next to the download with
     * a few headers.
     * <p/>
     * This method deletes the cached file and the info file ONLY if it
     * attempted a download and it failed to complete. It doesn't erase
     * anything if there's no download because the server returned a 404
     * or 304 or similar.
     *
     * @return An in-memory byte buffer input stream for the downloaded
     *   and locally cached file, or null if nothing was downloaded
     *   (including if it was a 304 Not-Modified status code.)
     */
    private InputStream downloadAndCache(
            String urlString,
            ITaskMonitor monitor,
            File cached,
            File info,
            @Nullable Header[] headers,
            @Nullable AtomicInteger outStatusCode)
                throws FileNotFoundException, IOException, CanceledByUserException {
        InputStream is = null;
        OutputStream os = null;

        int inc = 65536;
        int curr = 0;
        byte[] result = new byte[inc];

        try {
            Pair<InputStream, HttpResponse> r = UrlOpener.openUrl(urlString, monitor, headers);

            is = r.getFirst();
            HttpResponse response = r.getSecond();

            if (DEBUG) {
                System.out.println(String.format("%s : fetch: %s => %s",  //$NON-NLS-1$
                        urlString,
                        headers == null ? "" : Arrays.toString(headers),    //$NON-NLS-1$
                        response.getStatusLine()));
            }

            int code = response.getStatusLine().getStatusCode();

            if (outStatusCode != null) {
                outStatusCode.set(code);
            }

            if (code != HttpStatus.SC_OK) {
                // Only a 200 response code makes sense here.
                // Even the other 20x codes should not apply, e.g. no content or partial
                // content are not statuses we want to handle and should never happen.
                // (see http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 for list)
                return null;
            }

            os = new FileOutputStream(cached);

            int n;
            while ((n = is.read(result, curr, result.length - curr)) != -1) {
                if (os != null && n > 0) {
                    os.write(result, curr, n);
                }

                curr += n;

                if (os != null && curr > MAX_SMALL_FILE_SIZE) {
                    // If the file size exceeds our "small file size" threshold,
                    // stop caching. We don't want to fill the disk.
                    try {
                        os.close();
                    } catch (IOException ignore) {}
                    try {
                        cached.delete();
                        info.delete();
                    } catch (SecurityException ignore) {}
                    os = null;
                }
                if (curr == result.length) {
                    byte[] temp = new byte[curr + inc];
                    System.arraycopy(result, 0, temp, 0, curr);
                    result = temp;
                }
            }

            // Close the output stream, signaling it was stored properly.
            if (os != null) {
                try {
                    os.close();
                    os = null;

                    saveInfo(urlString, response, info);
                } catch (IOException ignore) {}
            }

            return new ByteArrayInputStream(result, 0, curr);

        } finally {
            if (is != null) {
                try {
                    is.close();
                } catch (IOException ignore) {}
            }
            if (os != null) {
                try {
                    os.close();
                } catch (IOException ignore) {}
                // If we get here with the output stream not null, it means there
                // was an issue and we don't want to keep that file. We'll try to
                // delete it.
                try {
                    cached.delete();
                    info.delete();
                } catch (SecurityException ignore) {}
            }
        }
    }

    /**
     * Saves part of the HTTP Response to the info file.
     */
    private void saveInfo(String urlString, HttpResponse response, File info) throws IOException {
        Properties props = new Properties();

        // we don't need the status code & URL right now.
        // Save it in case we want to have it later (e.g. to differentiate 200 and 404.)
        props.setProperty(KEY_URL, urlString);
        props.setProperty(KEY_STATUS_CODE,
                Integer.toString(response.getStatusLine().getStatusCode()));

        for (String name : INFO_HTTP_HEADERS) {
            Header h = response.getFirstHeader(name);
            if (h != null) {
                props.setProperty(name, h.getValue());
            }
        }

        FileOutputStream os = null;
        try {
            os = new FileOutputStream(info);
            props.store(os, "## Meta data for SDK Manager cache. Do not modify."); //$NON-NLS-1$
        } finally {
            if (os != null) {
                os.close();
            }
        }
    }

    /**
     * Reads the info properties file.
     * @return The properties found or null if there's no file or it can't be read.
     */
    private Properties readInfo(File info) {
        if (info.exists()) {
            Properties props = new Properties();

            InputStream is = null;
            try {
                is = new FileInputStream(info);
                props.load(is);
                return props;
            } catch (IOException ignore) {
            } finally {
                if (is != null) {
                    try {
                        is.close();
                    } catch (IOException ignore) {}
                }
            }
        }
        return null;
    }

    /**
     * Computes the cache filename for the given URL.
     * The filename uses the {@link #BIN_FILE_PREFIX}, the full URL string's hashcode and
     * a sanitized portion of the URL filename. The returned filename is never
     * more than 64 characters to ensure maximum file system compatibility.
     *
     * @param urlString The download URL.
     * @return A leaf filename for the cached download file.
     */
    private String getCacheFilename(String urlString) {
        String hash = String.format("%08x", urlString.hashCode());

        String leaf = urlString.toLowerCase(Locale.US);
        if (leaf.length() >= 2) {
            int index = urlString.lastIndexOf('/', leaf.length() - 2);
            leaf = urlString.substring(index + 1);
        }

        leaf = leaf.replaceAll("[^a-z0-9_-]+", "_");
        leaf = leaf.replaceAll("__+", "_");

        leaf = hash + '-' + leaf;
        int n = 64 - BIN_FILE_PREFIX.length();
        if (leaf.length() > n) {
            leaf = leaf.substring(0, n);
        }

        return BIN_FILE_PREFIX + leaf;
    }

    private String getInfoFilename(String cacheFilename) {
        return cacheFilename.replaceFirst(BIN_FILE_PREFIX, INFO_FILE_PREFIX);
    }
}