1#!/bin/bash 2# Copyright 2019 The TensorFlow Authors. All Rights Reserved. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# ============================================================================== 16 17# Utility script that handles downloading, extracting, and patching third-party 18# library dependencies for TensorFlow Lite for Microcontrollers. 19# Called with four arguments: 20# 1 - URL to download from. 21# 2 - MD5 checksum to verify the package's integrity. Use md5sum to create one. 22# 3 - Path to new folder to unpack the library into. 23# 4 - Optional patching action name. 24 25set -e 26 27# Patches the Ambiq Micro SDK to work around build issues. 28patch_am_sdk() { 29 local am_dir="${1}" 30 if [ ! -f ${am_dir}/VERSION.txt ]; then 31 echo "Could not find ${am_dir}, skipping AmbiqMicro SDK patch"; 32 return; 33 fi 34 35 local src_dir=${am_dir}/boards/apollo3_evb/examples/hello_world/gcc 36 local dest_dir=${am_dir}/boards/apollo3_evb/examples/hello_world/gcc_patched 37 38 rm -rf ${dest_dir} 39 mkdir ${dest_dir} 40 41 cp "${src_dir}/startup_gcc.c" "${dest_dir}/startup_gcc.c" 42 cp "${src_dir}/hello_world.ld" "${dest_dir}/apollo3evb.ld" 43 44 sed -i -e '114s/1024/1024\*20/g' "${dest_dir}/startup_gcc.c" 45 #sed -i -e 's/main/_main/g' "${dest_dir}/startup_gcc.c" 46 47 sed -i -e '3s/hello_world.ld/apollo3evb.ld/g' "${dest_dir}/apollo3evb.ld" 48 sed -i -e '3s/startup_gnu/startup_gcc/g' "${dest_dir}/apollo3evb.ld" 49 sed -i -e $'22s/\*(.text\*)/\*(.text\*)\\\n\\\n\\\t\/\* These are the C++ global constructors. Stick them all here and\\\n\\\t \* then walk through the array in main() calling them all.\\\n\\\t \*\/\\\n\\\t_init_array_start = .;\\\n\\\tKEEP (\*(SORT(.init_array\*)))\\\n\\\t_init_array_end = .;\\\n\\\n\\\t\/\* XXX Currently not doing anything for global destructors. \*\/\\\n/g' "${dest_dir}/apollo3evb.ld" 50 sed -i -e $'70s/} > SRAM/} > SRAM\\\n \/\* Add this to satisfy reference to symbol "end" from libnosys.a(sbrk.o)\\\n \* to denote the HEAP start.\\\n \*\/\\\n end = .;/g' "${dest_dir}/apollo3evb.ld" 51 52 # Add a delay after establishing serial connection 53 sed -ir -E $'s/ with serial\.Serial\(args\.port, args\.baud, timeout=12\) as ser:/ with serial.Serial(args.port, args.baud, timeout=12) as ser:\\\n # Patched.\\\n import time\\\n time.sleep(0.25)\\\n # End patch./g' "${am_dir}/tools/apollo3_scripts/uart_wired_update.py" 54 55 # Add CPP include guards to "am_hal_iom.h" 56 sed -i -e '57a\ 57 #ifdef __cplusplus // Patch\ 58 extern "C" {\ 59 #endif // End patch 60 ' "${am_dir}/mcu/apollo3/hal/am_hal_iom.h" 61 62 sed -i -e '836a\ 63 #ifdef __cplusplus // Patch\ 64 }\ 65 #endif // End patch 66 ' "${am_dir}/mcu/apollo3/hal/am_hal_iom.h" 67 68 echo "Finished preparing Apollo3 files" 69} 70 71# Fixes issues with KissFFT. 72patch_kissfft() { 73 sed -i -E $'s@#ifdef FIXED_POINT@// Patched automatically by download_dependencies.sh so default is 16 bit.\\\n#ifndef FIXED_POINT\\\n#define FIXED_POINT (16)\\\n#endif\\\n// End patch.\\\n\\\n#ifdef FIXED_POINT@g' tensorflow/lite/micro/tools/make/downloads/kissfft/kiss_fft.h 74 75 sed -i -E '/^#include <sys\/types.h>/d' tensorflow/lite/micro/tools/make/downloads/kissfft/kiss_fft.h 76 # Fix for https://github.com/mborgerding/kissfft/issues/20 77 sed -i -E $'s@#ifdef FIXED_POINT@#ifdef FIXED_POINT\\\n#include <stdint.h> /* Patched. */@g' tensorflow/lite/micro/tools/make/downloads/kissfft/kiss_fft.h 78 79 sed -i -E "s@#define KISS_FFT_MALLOC malloc@#define KISS_FFT_MALLOC(X) (void*)(0) /* Patched. */@g" tensorflow/lite/micro/tools/make/downloads/kissfft/kiss_fft.h 80 sed -i -E "s@#define KISS_FFT_FREE free@#define KISS_FFT_FREE(X) /* Patched. */@g" tensorflow/lite/micro/tools/make/downloads/kissfft/kiss_fft.h 81 sed -ir -E "s@(fprintf.*\);)@/* \1 */@g" tensorflow/lite/micro/tools/make/downloads/kissfft/tools/kiss_fftr.c 82 sed -ir -E "s@(exit.*\);)@return; /* \1 */@g" tensorflow/lite/micro/tools/make/downloads/kissfft/tools/kiss_fftr.c 83 echo "Finished patching kissfft" 84} 85 86# Create a header file containing an array with the first 10 images from the 87# CIFAR10 test dataset. 88patch_cifar10_dataset() { 89 xxd -l 30730 -i ${1}/test_batch.bin ${1}/../../../../examples/image_recognition_experimental/first_10_cifar_images.h 90 sed -i -E "s/unsigned char/const unsigned char/g" ${1}/../../../../examples/image_recognition_experimental/first_10_cifar_images.h 91} 92 93build_embarc_mli() { 94 make -j 4 -C ${1}/lib/make TCF_FILE=${2} 95} 96 97setup_zephyr() { 98 command -v virtualenv >/dev/null 2>&1 || { 99 echo >&2 "The required 'virtualenv' tool isn't installed. Try 'pip install virtualenv'."; exit 1; 100 } 101 virtualenv -p python3 ${1}/venv-zephyr 102 . ${1}/venv-zephyr/bin/activate 103 python ${1}/venv-zephyr/bin/pip install -r ${1}/scripts/requirements.txt 104 west init -m https://github.com/zephyrproject-rtos/zephyr.git 105 deactivate 106} 107 108# Main function handling the download, verify, extract, and patch process. 109download_and_extract() { 110 local usage="Usage: download_and_extract URL MD5 DIR [ACTION] [ACTION_PARAM]" 111 local url="${1:?${usage}}" 112 local expected_md5="${2:?${usage}}" 113 local dir="${3:?${usage}}" 114 local action=${4} 115 local action_param1=${5} # optional action parameter 116 local tempdir=$(mktemp -d) 117 local tempdir2=$(mktemp -d) 118 local tempfile=${tempdir}/temp_file 119 local curl_retries=5 120 121 # Destionation already downloaded. 122 if [ -d ${dir} ]; then 123 exit 0 124 fi 125 126 command -v curl >/dev/null 2>&1 || { 127 echo >&2 "The required 'curl' tool isn't installed. Try 'apt-get install curl'."; exit 1; 128 } 129 130 echo "downloading ${url}" >&2 131 mkdir -p "${dir}" 132 # We've been seeing occasional 56 errors from valid URLs, so set up a retry 133 # loop to attempt to recover from them. 134 for (( i=1; i<=$curl_retries; ++i )); do 135 # We have to use this approach because we normally halt the script when 136 # there's an error, and instead we want to catch errors so we can retry. 137 set +ex 138 curl -LsS --fail --retry 5 "${url}" > ${tempfile} 139 CURL_RESULT=$? 140 set -ex 141 142 # Was the command successful? If so, continue. 143 if [[ $CURL_RESULT -eq 0 ]]; then 144 break 145 fi 146 147 # Keep trying if we see the '56' error code. 148 if [[ ( $CURL_RESULT -ne 56 ) || ( $i -eq $curl_retries ) ]]; then 149 echo "Error $CURL_RESULT downloading '${url}'" 150 exit 1 151 fi 152 sleep 2 153 done 154 155 # Check that the file was downloaded correctly using a checksum. 156 DOWNLOADED_MD5=$(openssl dgst -md5 ${tempfile} | sed 's/.* //g') 157 if [ ${expected_md5} != ${DOWNLOADED_MD5} ]; then 158 echo "Checksum error for '${url}'. Expected ${expected_md5} but found ${DOWNLOADED_MD5}" 159 exit 1 160 fi 161 162 # delete anything after the '?' in a url that may mask true file extension 163 url=$(echo "${url}" | sed "s/\?.*//") 164 165 if [[ "${url}" == *gz ]]; then 166 tar -C "${dir}" --strip-components=1 -xzf ${tempfile} 167 elif [[ "${url}" == *tar.xz ]]; then 168 tar -C "${dir}" --strip-components=1 -xf ${tempfile} 169 elif [[ "${url}" == *bz2 ]]; then 170 curl -Ls "${url}" > ${tempdir}/tarred.bz2 171 tar -C "${dir}" --strip-components=1 -xjf ${tempfile} 172 elif [[ "${url}" == *zip ]]; then 173 unzip ${tempfile} -d ${tempdir2} 2>&1 1>/dev/null 174 # If the zip file contains nested directories, extract the files from the 175 # inner directory. 176 if [ $(find $tempdir2/* -maxdepth 0 | wc -l) = 1 ] && [ -d $tempdir2/* ]; then 177 # unzip has no strip components, so unzip to a temp dir, and move the 178 # files we want from the tempdir to destination. 179 cp -R ${tempdir2}/*/* ${dir}/ 180 else 181 cp -R ${tempdir2}/* ${dir}/ 182 fi 183 else 184 echo "Error unsupported archive type. Failed to extract tool after download." 185 exit 1 186 fi 187 rm -rf ${tempdir2} ${tempdir} 188 189 # Delete any potential BUILD files, which would interfere with Bazel builds. 190 find "${dir}" -type f -name '*BUILD' -delete 191 192 if [[ ${action} == "patch_am_sdk" ]]; then 193 patch_am_sdk ${dir} 194 elif [[ ${action} == "patch_kissfft" ]]; then 195 patch_kissfft ${dir} 196 elif [[ ${action} == "patch_cifar10_dataset" ]]; then 197 patch_cifar10_dataset ${dir} 198 elif [[ ${action} == "build_embarc_mli" ]]; then 199 if [[ "${action_param1}" == *.tcf ]]; then 200 cp ${action_param1} ${dir}/hw/arc.tcf 201 build_embarc_mli ${dir} ../../hw/arc.tcf 202 else 203 build_embarc_mli ${dir} ${action_param1} 204 fi 205 elif [[ ${action} == "setup_zephyr" ]]; then 206 setup_zephyr ${dir} 207 elif [[ ${action} ]]; then 208 echo "Unknown action '${action}'" 209 exit 1 210 fi 211} 212 213download_and_extract "$1" "$2" "$3" "$4" "$5" 214