1#!/bin/bash
2set -e
3
4scriptName="$(basename $0)"
5
6function usage() {
7  echo "NAME"
8  echo "  diagnose-build-failure.sh"
9  echo
10  echo "SYNOPSIS"
11  echo "  ./development/diagnose-build-failure/diagnose-build-failure.sh [--message <message>] [--timeout <seconds> ] '<tasks>'"
12  echo
13  echo "DESCRIPTION"
14  echo "  Attempts to identify why "'`'"./gradlew <tasks>"'`'" fails"
15  echo
16  echo "OPTIONS"
17  echo "--message <message>"
18  echo "  Replaces the requirement for "'`'"./gradlew <tasks>"'`'" to fail with the requirement that it produces the given message"
19  echo
20  echo "SAMPLE USAGE"
21  echo "  $0 assembleRelease # or any other arguments you would normally give to ./gradlew"
22  echo
23  echo "OUTPUT"
24  echo "  diagnose-build-failure will conclude one of the following:"
25  echo
26  echo "  A) Some state saved in memory by the Gradle daemon is triggering an error"
27  echo "  B) Your source files have been changed"
28  echo "     To (slowly) generate a simpler reproduction case, you can run simplify-build-failure.sh"
29  echo "  C) Some file in the out/ dir is triggering an error"
30  echo "     If this happens, $scriptName will identify which file(s) specifically"
31  echo "  D) The build is nondeterministic and/or affected by timestamps"
32  echo "  E) The build via gradlew actually passes"
33  exit 1
34}
35
36expectedMessage=""
37timeoutSeconds=""
38grepOptions=""
39while true; do
40  if [ "$#" -lt 1 ]; then
41    usage
42  fi
43  arg="$1"
44  shift
45  if [ "$arg" == "--message" ]; then
46    expectedMessage="$1"
47    shift
48    continue
49  fi
50  if [ "$arg" == "--timeout" ]; then
51    timeoutSeconds="$1"
52    shift
53    continue
54  fi
55
56  gradleArgs="$arg"
57  break
58done
59if [ "$gradleArgs" == "" ]; then
60  usage
61fi
62if [ "$timeoutSeconds" == "" ]; then
63  timeoutArg=""
64else
65  timeoutArg="--timeout $timeoutSeconds"
66fi
67# split Gradle arguments into options and tasks
68gradleOptions=""
69gradleTasks=""
70for arg in $gradleArgs; do
71  if [[ "$arg" == "-*" ]]; then
72    gradleOptions="$gradleOptions $arg"
73  else
74    gradleTasks="$gradleTasks $arg"
75  fi
76done
77
78if [ "$#" -gt 0 ]; then
79  echo "Unrecognized argument: $1" >&2
80  exit 1
81fi
82
83workingDir="$(pwd)"
84if [ ! -e "$workingDir/gradlew" ]; then
85  echo "Error; ./gradlew does not exist. Must cd to a dir containing a ./gradlew first" >&2
86  # so that this script knows which gradlew to use (in frameworks/support or frameworks/support/ui)
87  exit 1
88fi
89
90# resolve some paths
91scriptPath="$(cd $(dirname $0) && pwd)"
92vgrep="$scriptPath/impl/vgrep.sh"
93supportRoot="$(cd $scriptPath/../.. && pwd)"
94checkoutRoot="$(cd $supportRoot/../.. && pwd)"
95tempDir="$checkoutRoot/diagnose-build-failure/"
96if [ "$OUT_DIR" != "" ]; then
97  mkdir -p "$OUT_DIR"
98  OUT_DIR="$(cd $OUT_DIR && pwd)"
99  EFFECTIVE_OUT_DIR="$OUT_DIR"
100else
101  EFFECTIVE_OUT_DIR="$checkoutRoot/out"
102fi
103if [ "$DIST_DIR" != "" ]; then
104  mkdir -p "$DIST_DIR"
105  DIST_DIR="$(cd $DIST_DIR && pwd)"
106  EFFECTIVE_DIST_DIR=$DIST_DIR
107else
108  # If $DIST_DIR was unset, we leave it unset just in case setting it could affect the build
109  # However, we still need to keep track of where the files are going to go, so
110  # we set EFFECTIVE_DIST_DIR
111  EFFECTIVE_DIST_DIR="$EFFECTIVE_OUT_DIR/dist"
112fi
113COLOR_WHITE="\e[97m"
114COLOR_GREEN="\e[32m"
115
116function outputAdvice() {
117  adviceName="$1"
118  cd "$scriptPath"
119  adviceFilepath="classifications/${adviceName}.md"
120  echo >&2
121  echo "Advice ${scriptPath}/${adviceFilepath}:" >&2
122  echo >&2
123  cat "$adviceFilepath" >&2
124  exit 0
125}
126
127function checkStatusRepo() {
128  repo status >&2
129}
130
131function checkStatusGit() {
132  git status >&2
133  git log -1 >&2
134}
135
136function checkStatus() {
137  cd "$checkoutRoot"
138  if [ "-e" .repo ]; then
139    checkStatusRepo
140  else
141    checkStatusGit
142  fi
143}
144
145# echos a shell command for running the build in the current directory
146function getBuildCommand() {
147  if [ "$expectedMessage" == "" ]; then
148    testCommand="$* 2>&1"
149  else
150    # escape single quotes (end the previous quote, add an escaped quote, and start a new quote)
151    escapedMessage="$(echo "$expectedMessage" | sed "s/'/'\\\\''/g")"
152    testCommand="$* >log 2>&1; $vgrep '$escapedMessage' log $grepOptions"
153  fi
154  echo "$testCommand"
155}
156
157# Echos a shell command for testing the state in the current directory
158# Status can be inverted by the '--invert' flag
159# The dir of the state being tested is $testDir
160# The dir of the source code is $workingDir
161function getTestStateCommand() {
162  successStatus=0
163  failureStatus=1
164  if [[ "$1" == "--invert" ]]; then
165    successStatus=1
166    failureStatus=0
167    shift
168  fi
169
170  setupCommand="testDir=\$(pwd)
171$scriptPath/impl/restore-state.sh . $workingDir --move && cd $workingDir
172"
173  buildCommand="$*"
174  cleanupCommand="$scriptPath/impl/backup-state.sh \$testDir --move >/dev/null"
175
176  fullFiltererCommand="$setupCommand
177if $buildCommand >/dev/null 2>/dev/null; then
178  $cleanupCommand
179  exit $successStatus
180else
181  $cleanupCommand
182  exit $failureStatus
183fi"
184
185  echo "$fullFiltererCommand"
186}
187
188function runBuild() {
189  testCommand="$(getBuildCommand $*)"
190  cd "$workingDir"
191  echo Running $testCommand
192  if bash -c "$testCommand"; then
193    echo -e "$COLOR_WHITE"
194    echo
195    echo '`'$testCommand'`' succeeded
196    return 0
197  else
198    echo -e "$COLOR_WHITE"
199    echo
200    echo '`'$testCommand'`' failed
201    return 1
202  fi
203}
204
205function backupState() {
206  cd "$scriptPath"
207  backupDir="$1"
208  shift
209  ./impl/backup-state.sh "$backupDir" "$@"
210}
211
212function restoreState() {
213  cd "$scriptPath"
214  backupDir="$1"
215  ./impl/restore-state.sh "$backupDir"
216}
217
218function clearState() {
219  restoreState /dev/null
220}
221
222echo >&2
223echo "diagnose-build-failure making sure that we can reproduce the build failure" >&2
224if runBuild ./gradlew -Pandroidx.summarizeStderr $gradleArgs; then
225  echo >&2
226  echo "This script failed to reproduce the build failure." >&2
227  outputAdvice "subsequent-success"
228else
229  echo >&2
230  echo "Reproduced build failure" >&2
231fi
232
233if [ "$expectedMessage" == "" ]; then
234  summaryLog="$EFFECTIVE_DIST_DIR/logs/error_summary.log"
235  echo
236  echo "No failure message specified. Computing appropriate failure message from $summaryLog"
237  echo
238  longestLine="$(awk '{ if (length($0) > maxLength) {maxLength = length($0); longestLine = $0} } END { print longestLine }' $summaryLog)"
239  echo "Longest line:"
240  echo
241  echo "$longestLine"
242  echo
243  grepOptions="-F" # interpret grep query as a fixed string, not a regex
244  if grep $grepOptions "$longestLine" "$summaryLog" >/dev/null 2>/dev/null; then
245    echo "We will use this as the message to test for"
246    echo
247    expectedMessage="$longestLine"
248  else
249    echo "The identified line could not be found in the summary log via grep. Is it possible that diagnose-build-failure did not correctly escape the message?"
250    exit 1
251  fi
252fi
253
254echo
255echo "diagnose-build-failure stopping the Gradle Daemon and rebuilding" >&2
256cd "$supportRoot"
257./gradlew --stop || true
258if runBuild ./gradlew --no-daemon $gradleArgs; then
259  echo >&2
260  echo "The build passed when disabling the Gradle Daemon" >&2
261  outputAdvice "memory-state"
262else
263  echo >&2
264  echo "The build failed even with the Gradle Daemon disabled." >&2
265  echo "This may mean that there is state stored in a file somewhere, triggering the build to fail." >&2
266  echo "We will investigate the possibility of saved state next." >&2
267  echo >&2
268  # We're going to immediately overwrite the user's current state,
269  # so we can simply move the current state into $tempDir/prev rather than copying it
270  backupState "$tempDir/prev" --move
271fi
272
273echo >&2
274echo "Checking whether a clean build passes" >&2
275clearState
276backupState "$tempDir/empty"
277successState="$tempDir/empty"
278if runBuild ./gradlew --no-daemon $gradleArgs; then
279  echo >&2
280  echo "The clean build passed, so we can now investigate what cached state is triggering this build to fail." >&2
281  backupState "$tempDir/clean"
282else
283  echo >&2
284  echo "The clean build also reproduced the issue." >&2
285  outputAdvice "clean-error"
286  echo "Checking the status of the checkout:" >&2
287  checkStatus
288  exit 1
289fi
290
291echo >&2
292echo "Checking whether a second build passes when starting from the output of the first clean build" >&2
293if runBuild ./gradlew --no-daemon $gradleArgs; then
294  echo >&2
295  echo "The next build after the clean build passed, so we can use the output of the first clean build as the successful state to compare against" >&2
296  successState="$tempDir/clean"
297else
298  echo >&2
299  echo "The next build after the clean build failed." >&2
300  echo "Although this is unexpected, we should still be able to diagnose it." >&2
301  echo "This might be slower than normal, though, because it may require us to rebuild more things more often" >&2
302fi
303
304echo >&2
305echo "Next we'll double-check that after restoring the failing state, the build fails" >&2
306restoreState "$tempDir/prev"
307if runBuild ./gradlew --no-daemon $gradleArgs; then
308  echo >&2
309  echo "After restoring the saved state, the build passed." >&2
310  outputAdvice "unknown-state"
311  exit 1
312else
313  echo >&2
314  echo "After restoring the saved state, the build failed. This confirms that this script is successfully saving and restoring the relevant state" >&2
315fi
316
317# Ask diff-filterer.py to run a binary search to determine the minimum set of tasks that must be passed to reproduce this error
318# (it's possible that the caller passed more tasks than needed, particularly if the caller is a script)
319requiredTasksDir="$tempDir/requiredTasks"
320function determineMinimalSetOfRequiredTasks() {
321  echo Calculating the list of tasks to run
322  allTasksLog="$tempDir/tasks.log"
323  restoreState "$successState"
324  rm -f "$allTasksLog"
325  bash -c "cd $workingDir && ./gradlew --no-daemon --dry-run $gradleArgs > $allTasksLog 2>&1" || true
326
327  # process output and split into files
328  taskListFile="$tempDir/tasks.list"
329  cat "$allTasksLog" | grep '^:' | sed 's/ .*//' > "$taskListFile"
330  requiredTasksWork="$tempDir/requiredTasksWork"
331  rm -rf "$requiredTasksWork"
332  cp -r "$tempDir/prev" "$requiredTasksWork"
333  mkdir -p "$requiredTasksWork/tasks"
334  bash -c "cd $requiredTasksWork/tasks && split -l 1 '$taskListFile'"
335  # also include the original tasks in case either we failed to compute the list of tasks (due to the build failing during project configuration) or there are too many tasks to fit in one command line invocation
336  echo "$gradleTasks" > "$requiredTasksWork/tasks/givenTasks"
337
338  rm -rf "$requiredTasksDir"
339  # Build the command for passing to diff-filterer.
340  # We call xargs because the full set of tasks might be too long for the shell, and xargs will
341  # split into multiple gradlew invocations if needed.
342  # We also cd into the tasks/ dir before calling 'cat' to avoid reaching its argument length limit.
343  # note that the variable "$testDir" gets set by $getTestStateCommand
344  buildCommand="$(getBuildCommand "rm -f log && (cd \$testDir/tasks && cat *) | xargs --no-run-if-empty ./gradlew $gradleOptions")"
345
346  # command for moving state, running build, and moving state back
347  fullFiltererCommand="$(getTestStateCommand --invert $buildCommand)"
348
349  if $supportRoot/development/file-utils/diff-filterer.py $timeoutArg --work-path "$tempDir" "$requiredTasksWork" "$tempDir/prev"  "$fullFiltererCommand"; then
350    echo diff-filterer successfully identified a minimal set of required tasks. Saving into $requiredTasksDir >&2
351    cp -r "$tempDir/bestResults/tasks" "$requiredTasksDir"
352  else
353    echo diff-filterer was unable to identify a minimal set of tasks required to reproduce the error >&2
354    exit 1
355  fi
356}
357determineMinimalSetOfRequiredTasks
358# update variables
359gradleTasks="$(cat $requiredTasksDir/*)"
360gradleArgs="$gradleOptions $gradleTasks"
361
362# Now ask diff-filterer.py to run a binary search to determine what the relevant differences are between "$tempDir/prev" and "$tempDir/clean"
363echo >&2
364echo "Binary-searching the contents of the two output directories until the relevant differences are identified." >&2
365echo "This may take a while."
366echo >&2
367
368# command for running a build
369buildCommand="$(getBuildCommand "./gradlew --no-daemon $gradleArgs")"
370# command for moving state, running build, and moving state back
371fullFiltererCommand="$(getTestStateCommand --invert $buildCommand)"
372
373if $supportRoot/development/file-utils/diff-filterer.py $timeoutArg --assume-input-states-are-correct --work-path $tempDir $tempDir/prev $successState "$fullFiltererCommand"; then
374  echo >&2
375  echo "There should be something wrong with the above file state" >&2
376  echo "Hopefully the output from diff-filterer.py above is enough information for you to figure out what is wrong" >&2
377  echo "If not, you could ask a team member about your original error message and see if they have any ideas" >&2
378else
379  echo >&2
380  echo "Something went wrong running diff-filterer.py" >&2
381  echo "Maybe that means the build is nondeterministic" >&2
382  echo "Maybe that means that there's something wrong with this script ($0)" >&2
383fi
384