• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1from __future__ import absolute_import
2import os, signal, subprocess, sys
3import re
4import platform
5import tempfile
6
7import lit.ShUtil as ShUtil
8import lit.Test as Test
9import lit.util
10from lit.util import to_bytes, to_string
11
12class InternalShellError(Exception):
13    def __init__(self, command, message):
14        self.command = command
15        self.message = message
16
17kIsWindows = platform.system() == 'Windows'
18
19# Don't use close_fds on Windows.
20kUseCloseFDs = not kIsWindows
21
22# Use temporary files to replace /dev/null on Windows.
23kAvoidDevNull = kIsWindows
24
25class ShellEnvironment(object):
26
27    """Mutable shell environment containing things like CWD and env vars.
28
29    Environment variables are not implemented, but cwd tracking is.
30    """
31
32    def __init__(self, cwd, env):
33        self.cwd = cwd
34        self.env = env
35
36def executeShCmd(cmd, shenv, results):
37    if isinstance(cmd, ShUtil.Seq):
38        if cmd.op == ';':
39            res = executeShCmd(cmd.lhs, shenv, results)
40            return executeShCmd(cmd.rhs, shenv, results)
41
42        if cmd.op == '&':
43            raise InternalShellError(cmd,"unsupported shell operator: '&'")
44
45        if cmd.op == '||':
46            res = executeShCmd(cmd.lhs, shenv, results)
47            if res != 0:
48                res = executeShCmd(cmd.rhs, shenv, results)
49            return res
50
51        if cmd.op == '&&':
52            res = executeShCmd(cmd.lhs, shenv, results)
53            if res is None:
54                return res
55
56            if res == 0:
57                res = executeShCmd(cmd.rhs, shenv, results)
58            return res
59
60        raise ValueError('Unknown shell command: %r' % cmd.op)
61    assert isinstance(cmd, ShUtil.Pipeline)
62
63    # Handle shell builtins first.
64    if cmd.commands[0].args[0] == 'cd':
65        # Update the cwd in the environment.
66        if len(cmd.commands[0].args) != 2:
67            raise ValueError('cd supports only one argument')
68        newdir = cmd.commands[0].args[1]
69        if os.path.isabs(newdir):
70            shenv.cwd = newdir
71        else:
72            shenv.cwd = os.path.join(shenv.cwd, newdir)
73        return 0
74
75    procs = []
76    input = subprocess.PIPE
77    stderrTempFiles = []
78    opened_files = []
79    named_temp_files = []
80    # To avoid deadlock, we use a single stderr stream for piped
81    # output. This is null until we have seen some output using
82    # stderr.
83    for i,j in enumerate(cmd.commands):
84        # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
85        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
86        # from a file are represented with a list [file, mode, file-object]
87        # where file-object is initially None.
88        redirects = [(0,), (1,), (2,)]
89        for r in j.redirects:
90            if r[0] == ('>',2):
91                redirects[2] = [r[1], 'w', None]
92            elif r[0] == ('>>',2):
93                redirects[2] = [r[1], 'a', None]
94            elif r[0] == ('>&',2) and r[1] in '012':
95                redirects[2] = redirects[int(r[1])]
96            elif r[0] == ('>&',) or r[0] == ('&>',):
97                redirects[1] = redirects[2] = [r[1], 'w', None]
98            elif r[0] == ('>',):
99                redirects[1] = [r[1], 'w', None]
100            elif r[0] == ('>>',):
101                redirects[1] = [r[1], 'a', None]
102            elif r[0] == ('<',):
103                redirects[0] = [r[1], 'r', None]
104            else:
105                raise InternalShellError(j,"Unsupported redirect: %r" % (r,))
106
107        # Map from the final redirections to something subprocess can handle.
108        final_redirects = []
109        for index,r in enumerate(redirects):
110            if r == (0,):
111                result = input
112            elif r == (1,):
113                if index == 0:
114                    raise InternalShellError(j,"Unsupported redirect for stdin")
115                elif index == 1:
116                    result = subprocess.PIPE
117                else:
118                    result = subprocess.STDOUT
119            elif r == (2,):
120                if index != 2:
121                    raise InternalShellError(j,"Unsupported redirect on stdout")
122                result = subprocess.PIPE
123            else:
124                if r[2] is None:
125                    if kAvoidDevNull and r[0] == '/dev/null':
126                        r[2] = tempfile.TemporaryFile(mode=r[1])
127                    else:
128                        # Make sure relative paths are relative to the cwd.
129                        redir_filename = os.path.join(shenv.cwd, r[0])
130                        r[2] = open(redir_filename, r[1])
131                    # Workaround a Win32 and/or subprocess bug when appending.
132                    #
133                    # FIXME: Actually, this is probably an instance of PR6753.
134                    if r[1] == 'a':
135                        r[2].seek(0, 2)
136                    opened_files.append(r[2])
137                result = r[2]
138            final_redirects.append(result)
139
140        stdin, stdout, stderr = final_redirects
141
142        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
143        # stderr on a pipe and treat it as stdout.
144        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
145            stderr = subprocess.PIPE
146            stderrIsStdout = True
147        else:
148            stderrIsStdout = False
149
150            # Don't allow stderr on a PIPE except for the last
151            # process, this could deadlock.
152            #
153            # FIXME: This is slow, but so is deadlock.
154            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
155                stderr = tempfile.TemporaryFile(mode='w+b')
156                stderrTempFiles.append((i, stderr))
157
158        # Resolve the executable path ourselves.
159        args = list(j.args)
160        executable = lit.util.which(args[0], shenv.env['PATH'])
161        if not executable:
162            raise InternalShellError(j, '%r: command not found' % j.args[0])
163
164        # Replace uses of /dev/null with temporary files.
165        if kAvoidDevNull:
166            for i,arg in enumerate(args):
167                if arg == "/dev/null":
168                    f = tempfile.NamedTemporaryFile(delete=False)
169                    f.close()
170                    named_temp_files.append(f.name)
171                    args[i] = f.name
172
173        try:
174            procs.append(subprocess.Popen(args, cwd=shenv.cwd,
175                                          executable = executable,
176                                          stdin = stdin,
177                                          stdout = stdout,
178                                          stderr = stderr,
179                                          env = shenv.env,
180                                          close_fds = kUseCloseFDs))
181        except OSError as e:
182            raise InternalShellError(j, 'Could not create process due to {}'.format(e))
183
184        # Immediately close stdin for any process taking stdin from us.
185        if stdin == subprocess.PIPE:
186            procs[-1].stdin.close()
187            procs[-1].stdin = None
188
189        # Update the current stdin source.
190        if stdout == subprocess.PIPE:
191            input = procs[-1].stdout
192        elif stderrIsStdout:
193            input = procs[-1].stderr
194        else:
195            input = subprocess.PIPE
196
197    # Explicitly close any redirected files. We need to do this now because we
198    # need to release any handles we may have on the temporary files (important
199    # on Win32, for example). Since we have already spawned the subprocess, our
200    # handles have already been transferred so we do not need them anymore.
201    for f in opened_files:
202        f.close()
203
204    # FIXME: There is probably still deadlock potential here. Yawn.
205    procData = [None] * len(procs)
206    procData[-1] = procs[-1].communicate()
207
208    for i in range(len(procs) - 1):
209        if procs[i].stdout is not None:
210            out = procs[i].stdout.read()
211        else:
212            out = ''
213        if procs[i].stderr is not None:
214            err = procs[i].stderr.read()
215        else:
216            err = ''
217        procData[i] = (out,err)
218
219    # Read stderr out of the temp files.
220    for i,f in stderrTempFiles:
221        f.seek(0, 0)
222        procData[i] = (procData[i][0], f.read())
223
224    def to_string(bytes):
225        if isinstance(bytes, str):
226            return bytes
227        return bytes.encode('utf-8')
228
229    exitCode = None
230    for i,(out,err) in enumerate(procData):
231        res = procs[i].wait()
232        # Detect Ctrl-C in subprocess.
233        if res == -signal.SIGINT:
234            raise KeyboardInterrupt
235
236        # Ensure the resulting output is always of string type.
237        try:
238            out = to_string(out.decode('utf-8'))
239        except:
240            out = str(out)
241        try:
242            err = to_string(err.decode('utf-8'))
243        except:
244            err = str(err)
245
246        results.append((cmd.commands[i], out, err, res))
247        if cmd.pipe_err:
248            # Python treats the exit code as a signed char.
249            if exitCode is None:
250                exitCode = res
251            elif res < 0:
252                exitCode = min(exitCode, res)
253            else:
254                exitCode = max(exitCode, res)
255        else:
256            exitCode = res
257
258    # Remove any named temporary files we created.
259    for f in named_temp_files:
260        try:
261            os.remove(f)
262        except OSError:
263            pass
264
265    if cmd.negate:
266        exitCode = not exitCode
267
268    return exitCode
269
270def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
271    cmds = []
272    for ln in commands:
273        try:
274            cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
275                                        test.config.pipefail).parse())
276        except:
277            return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln)
278
279    cmd = cmds[0]
280    for c in cmds[1:]:
281        cmd = ShUtil.Seq(cmd, '&&', c)
282
283    results = []
284    try:
285        shenv = ShellEnvironment(cwd, test.config.environment)
286        exitCode = executeShCmd(cmd, shenv, results)
287    except InternalShellError:
288        e = sys.exc_info()[1]
289        exitCode = 127
290        results.append((e.command, '', e.message, exitCode))
291
292    out = err = ''
293    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
294        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
295        out += 'Command %d Result: %r\n' % (i, res)
296        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
297        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
298
299    return out, err, exitCode
300
301def executeScript(test, litConfig, tmpBase, commands, cwd):
302    bashPath = litConfig.getBashPath();
303    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
304    script = tmpBase + '.script'
305    if isWin32CMDEXE:
306        script += '.bat'
307
308    # Write script file
309    mode = 'w'
310    if litConfig.isWindows and not isWin32CMDEXE:
311      mode += 'b'  # Avoid CRLFs when writing bash scripts.
312    f = open(script, mode)
313    if isWin32CMDEXE:
314        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
315    else:
316        if test.config.pipefail:
317            f.write('set -o pipefail;')
318        f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
319    f.write('\n')
320    f.close()
321
322    if isWin32CMDEXE:
323        command = ['cmd','/c', script]
324    else:
325        if bashPath:
326            command = [bashPath, script]
327        else:
328            command = ['/bin/sh', script]
329        if litConfig.useValgrind:
330            # FIXME: Running valgrind on sh is overkill. We probably could just
331            # run on clang with no real loss.
332            command = litConfig.valgrindArgs + command
333
334    return lit.util.executeCommand(command, cwd=cwd,
335                                   env=test.config.environment)
336
337def parseIntegratedTestScriptCommands(source_path):
338    """
339    parseIntegratedTestScriptCommands(source_path) -> commands
340
341    Parse the commands in an integrated test script file into a list of
342    (line_number, command_type, line).
343    """
344
345    # This code is carefully written to be dual compatible with Python 2.5+ and
346    # Python 3 without requiring input files to always have valid codings. The
347    # trick we use is to open the file in binary mode and use the regular
348    # expression library to find the commands, with it scanning strings in
349    # Python2 and bytes in Python3.
350    #
351    # Once we find a match, we do require each script line to be decodable to
352    # UTF-8, so we convert the outputs to UTF-8 before returning. This way the
353    # remaining code can work with "strings" agnostic of the executing Python
354    # version.
355
356    keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.']
357    keywords_re = re.compile(
358        to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
359
360    f = open(source_path, 'rb')
361    try:
362        # Read the entire file contents.
363        data = f.read()
364
365        # Ensure the data ends with a newline.
366        if not data.endswith(to_bytes('\n')):
367            data = data + to_bytes('\n')
368
369        # Iterate over the matches.
370        line_number = 1
371        last_match_position = 0
372        for match in keywords_re.finditer(data):
373            # Compute the updated line number by counting the intervening
374            # newlines.
375            match_position = match.start()
376            line_number += data.count(to_bytes('\n'), last_match_position,
377                                      match_position)
378            last_match_position = match_position
379
380            # Convert the keyword and line to UTF-8 strings and yield the
381            # command. Note that we take care to return regular strings in
382            # Python 2, to avoid other code having to differentiate between the
383            # str and unicode types.
384            keyword,ln = match.groups()
385            yield (line_number, to_string(keyword[:-1].decode('utf-8')),
386                   to_string(ln.decode('utf-8')))
387    finally:
388        f.close()
389
390
391def parseIntegratedTestScript(test, normalize_slashes=False,
392                              extra_substitutions=[], require_script=True):
393    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
394    script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
395    and 'UNSUPPORTED' information. The RUN lines also will have variable
396    substitution performed. If 'require_script' is False an empty script may be
397    returned. This can be used for test formats where the actual script is
398    optional or ignored.
399    """
400
401    # Get the temporary location, this is always relative to the test suite
402    # root, not test source root.
403    #
404    # FIXME: This should not be here?
405    sourcepath = test.getSourcePath()
406    sourcedir = os.path.dirname(sourcepath)
407    execpath = test.getExecPath()
408    execdir,execbase = os.path.split(execpath)
409    tmpDir = os.path.join(execdir, 'Output')
410    tmpBase = os.path.join(tmpDir, execbase)
411
412    # Normalize slashes, if requested.
413    if normalize_slashes:
414        sourcepath = sourcepath.replace('\\', '/')
415        sourcedir = sourcedir.replace('\\', '/')
416        tmpDir = tmpDir.replace('\\', '/')
417        tmpBase = tmpBase.replace('\\', '/')
418
419    # We use #_MARKER_# to hide %% while we do the other substitutions.
420    substitutions = list(extra_substitutions)
421    substitutions.extend([('%%', '#_MARKER_#')])
422    substitutions.extend(test.config.substitutions)
423    substitutions.extend([('%s', sourcepath),
424                          ('%S', sourcedir),
425                          ('%p', sourcedir),
426                          ('%{pathsep}', os.pathsep),
427                          ('%t', tmpBase + '.tmp'),
428                          ('%T', tmpDir),
429                          ('#_MARKER_#', '%')])
430
431    # "%/[STpst]" should be normalized.
432    substitutions.extend([
433            ('%/s', sourcepath.replace('\\', '/')),
434            ('%/S', sourcedir.replace('\\', '/')),
435            ('%/p', sourcedir.replace('\\', '/')),
436            ('%/t', tmpBase.replace('\\', '/') + '.tmp'),
437            ('%/T', tmpDir.replace('\\', '/')),
438            ])
439
440    # Collect the test lines from the script.
441    script = []
442    requires = []
443    unsupported = []
444    for line_number, command_type, ln in \
445            parseIntegratedTestScriptCommands(sourcepath):
446        if command_type == 'RUN':
447            # Trim trailing whitespace.
448            ln = ln.rstrip()
449
450            # Substitute line number expressions
451            ln = re.sub('%\(line\)', str(line_number), ln)
452            def replace_line_number(match):
453                if match.group(1) == '+':
454                    return str(line_number + int(match.group(2)))
455                if match.group(1) == '-':
456                    return str(line_number - int(match.group(2)))
457            ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln)
458
459            # Collapse lines with trailing '\\'.
460            if script and script[-1][-1] == '\\':
461                script[-1] = script[-1][:-1] + ln
462            else:
463                script.append(ln)
464        elif command_type == 'XFAIL':
465            test.xfails.extend([s.strip() for s in ln.split(',')])
466        elif command_type == 'REQUIRES':
467            requires.extend([s.strip() for s in ln.split(',')])
468        elif command_type == 'UNSUPPORTED':
469            unsupported.extend([s.strip() for s in ln.split(',')])
470        elif command_type == 'END':
471            # END commands are only honored if the rest of the line is empty.
472            if not ln.strip():
473                break
474        else:
475            raise ValueError("unknown script command type: %r" % (
476                    command_type,))
477
478    # Apply substitutions to the script.  Allow full regular
479    # expression syntax.  Replace each matching occurrence of regular
480    # expression pattern a with substitution b in line ln.
481    def processLine(ln):
482        # Apply substitutions
483        for a,b in substitutions:
484            if kIsWindows:
485                b = b.replace("\\","\\\\")
486            ln = re.sub(a, b, ln)
487
488        # Strip the trailing newline and any extra whitespace.
489        return ln.strip()
490    script = [processLine(ln)
491              for ln in script]
492
493    # Verify the script contains a run line.
494    if require_script and not script:
495        return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!")
496
497    # Check for unterminated run lines.
498    if script and script[-1][-1] == '\\':
499        return lit.Test.Result(Test.UNRESOLVED,
500                               "Test has unterminated run lines (with '\\')")
501
502    # Check that we have the required features:
503    missing_required_features = [f for f in requires
504                                 if f not in test.config.available_features]
505    if missing_required_features:
506        msg = ', '.join(missing_required_features)
507        return lit.Test.Result(Test.UNSUPPORTED,
508                               "Test requires the following features: %s" % msg)
509    unsupported_features = [f for f in unsupported
510                            if f in test.config.available_features]
511    if unsupported_features:
512        msg = ', '.join(unsupported_features)
513        return lit.Test.Result(Test.UNSUPPORTED,
514                    "Test is unsupported with the following features: %s" % msg)
515
516    return script,tmpBase,execdir
517
518def _runShTest(test, litConfig, useExternalSh,
519                   script, tmpBase, execdir):
520    # Create the output directory if it does not already exist.
521    lit.util.mkdir_p(os.path.dirname(tmpBase))
522
523    if useExternalSh:
524        res = executeScript(test, litConfig, tmpBase, script, execdir)
525    else:
526        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
527    if isinstance(res, lit.Test.Result):
528        return res
529
530    out,err,exitCode = res
531    if exitCode == 0:
532        status = Test.PASS
533    else:
534        status = Test.FAIL
535
536    # Form the output log.
537    output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % (
538        '\n'.join(script), exitCode)
539
540    # Append the outputs, if present.
541    if out:
542        output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
543    if err:
544        output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
545
546    return lit.Test.Result(status, output)
547
548
549def executeShTest(test, litConfig, useExternalSh,
550                  extra_substitutions=[]):
551    if test.config.unsupported:
552        return (Test.UNSUPPORTED, 'Test is unsupported')
553
554    res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
555    if isinstance(res, lit.Test.Result):
556        return res
557    if litConfig.noExecute:
558        return lit.Test.Result(Test.PASS)
559
560    script, tmpBase, execdir = res
561    return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
562
563