1#!/bin/bash 2# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# ============================================================================== 16 17set -e 18 19DOWNLOADS_DIR=tensorflow/contrib/makefile/downloads 20BZL_FILE_PATH=tensorflow/workspace.bzl 21 22# Ensure it is being run from repo root 23if [ ! -f $BZL_FILE_PATH ]; then 24 echo "Could not find ${BZL_FILE_PATH}": 25 echo "Likely you are not running this from the root directory of the repository."; 26 exit 1; 27fi 28 29EIGEN_URL="$(grep -o 'http.*bitbucket.org/eigen/eigen/get/.*tar\.gz' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)" 30GEMMLOWP_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)" 31GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.gz" 32NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" 33 34# Note: The protobuf repo needs to be cloned due to its submodules. 35# These variables contain the GitHub repo and the sha, from `tensorflow/workspace.bzl`, 36# from which to clone it from and checkout to. 37readonly PROTOBUF_REPO="https://github.com/protocolbuffers/protobuf.git" 38readonly PROTOBUF_TAG="$(grep -o 'https://github.com/protocolbuffers/protobuf/archive/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1 | awk '{print substr($0, index($0, "archive") + 8, index($0, "tar") - index($0, "archive") - 9) }')" 39 40# TODO (yongtang): Replace the following with 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' once 41# the archive has been propagated in mirror.bazel.build. 42RE2_URL="$(grep -o 'https://github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" 43FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" 44DOUBLE_CONVERSION_URL="$(grep -o "https.*google/double-conversion.*\.zip" "${BZL_FILE_PATH}" | head -n1)" 45ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)" 46CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)" 47 48# Required for TensorFlow Lite Flex runtime. 49FARMHASH_URL="https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz" 50FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/1f5eae5d6a135ff6811724f6c57f911d1f46bb15.tar.gz" 51 52# TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64, 53# so work around it by patching the source. 54replace_by_sed() { 55 local regex="${1}" 56 shift 57 # Detect the version of sed by the return value of "--version" flag. GNU-sed 58 # supports "--version" while BSD-sed doesn't. 59 if ! sed --version >/dev/null 2>&1; then 60 # BSD-sed. 61 sed -i '' -e "${regex}" "$@" 62 else 63 # GNU-sed. 64 sed -i -e "${regex}" "$@" 65 fi 66} 67 68download_and_extract() { 69 local usage="Usage: download_and_extract URL DIR" 70 local url="${1:?${usage}}" 71 local dir="${2:?${usage}}" 72 echo "downloading ${url}" >&2 73 mkdir -p "${dir}" 74 if [[ "${url}" == *gz ]]; then 75 curl -Ls "${url}" | tar -C "${dir}" --strip-components=1 -xz 76 elif [[ "${url}" == *zip ]]; then 77 tempdir=$(mktemp -d) 78 tempdir2=$(mktemp -d) 79 if [[ "$OSTYPE" == "darwin"* ]]; then 80 # macOS (AKA darwin) doesn't have wget. 81 (cd "${tempdir}"; curl --remote-name --silent --location "${url}") 82 else 83 wget -P "${tempdir}" "${url}" 84 fi 85 unzip "${tempdir}"/* -d "${tempdir2}" 86 # unzip has no strip components, so unzip to a temp dir, and move the files 87 # we want from the tempdir to destination. 88 cp -R "${tempdir2}"/*/* "${dir}"/ 89 rm -rf "${tempdir2}" "${tempdir}" 90 fi 91 92 # Delete any potential BUILD files, which would interfere with Bazel builds. 93 find "${dir}" -type f -name '*BUILD' -delete 94} 95 96function clone_repository() { 97 local repo_url="${1}" 98 local destination_directory="${2}" 99 local commit_sha="${3}" 100 101 if [[ -d "${destination_directory}" ]]; then 102 rm -rf "${destination_directory}" 103 fi 104 105 git clone "${repo_url}" "${destination_directory}" 106 107 pushd "$(pwd)" 1>/dev/null 108 109 cd "${destination_directory}" 110 111 if [[ -n "${commit_sha}" ]]; then 112 git checkout "${PROTOBUF_TAG}" 113 fi 114 115 git submodule update --init 116 117 popd 1>/dev/null 118} 119 120download_and_extract "${EIGEN_URL}" "${DOWNLOADS_DIR}/eigen" 121download_and_extract "${GEMMLOWP_URL}" "${DOWNLOADS_DIR}/gemmlowp" 122download_and_extract "${GOOGLETEST_URL}" "${DOWNLOADS_DIR}/googletest" 123download_and_extract "${NSYNC_URL}" "${DOWNLOADS_DIR}/nsync" 124download_and_extract "${RE2_URL}" "${DOWNLOADS_DIR}/re2" 125download_and_extract "${FFT2D_URL}" "${DOWNLOADS_DIR}/fft2d" 126download_and_extract "${DOUBLE_CONVERSION_URL}" "${DOWNLOADS_DIR}/double_conversion" 127download_and_extract "${ABSL_URL}" "${DOWNLOADS_DIR}/absl" 128download_and_extract "${CUB_URL}" "${DOWNLOADS_DIR}/cub/external/cub_archive" 129 130# Required for TensorFlow Lite Flex runtime. 131download_and_extract "${FARMHASH_URL}" "${DOWNLOADS_DIR}/farmhash" 132download_and_extract "${FLATBUFFERS_URL}" "${DOWNLOADS_DIR}/flatbuffers" 133 134clone_repository "${PROTOBUF_REPO}" "${DOWNLOADS_DIR}/protobuf" "${PROTOBUF_TAG}" 135 136replace_by_sed 's#static uint32x4_t p4ui_CONJ_XOR = vld1q_u32( conj_XOR_DATA );#static uint32x4_t p4ui_CONJ_XOR; // = vld1q_u32( conj_XOR_DATA ); - Removed by script#' \ 137 "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" 138replace_by_sed 's#static uint32x2_t p2ui_CONJ_XOR = vld1_u32( conj_XOR_DATA );#static uint32x2_t p2ui_CONJ_XOR;// = vld1_u32( conj_XOR_DATA ); - Removed by scripts#' \ 139 "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" 140replace_by_sed 's#static uint64x2_t p2ul_CONJ_XOR = vld1q_u64( p2ul_conj_XOR_DATA );#static uint64x2_t p2ul_CONJ_XOR;// = vld1q_u64( p2ul_conj_XOR_DATA ); - Removed by script#' \ 141 "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/arch/NEON/Complex.h" 142# TODO(satok): Remove this once protobuf/autogen.sh is fixed. 143replace_by_sed 's#https://googlemock.googlecode.com/files/gmock-1.7.0.zip#http://download.tensorflow.org/deps/gmock-1.7.0.zip#' \ 144 "${DOWNLOADS_DIR}/protobuf/autogen.sh" 145cat "third_party/eigen3/gebp_neon.patch" | patch "${DOWNLOADS_DIR}/eigen/Eigen/src/Core/products/GeneralBlockPanelKernel.h" 146 147echo "download_dependencies.sh completed successfully." >&2 148