• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2"""Test LZ4 interoperability between versions"""
3
4#
5# Copyright (C) 2011-present, Takayuki Matsuoka
6# All rights reserved.
7# GPL v2 License
8#
9
10import glob
11import subprocess
12import filecmp
13import os
14import shutil
15import sys
16import hashlib
17
18repo_url = 'https://github.com/lz4/lz4.git'
19tmp_dir_name = 'tests/versionsTest'
20make_cmd = 'make'
21git_cmd = 'git'
22test_dat_src = 'README.md'
23test_dat = 'test_dat'
24head = 'v999'
25
26def proc(cmd_args, pipe=True, dummy=False):
27    if dummy:
28        return
29    if pipe:
30        subproc = subprocess.Popen(cmd_args,
31                                   stdout=subprocess.PIPE,
32                                   stderr=subprocess.PIPE)
33    else:
34        subproc = subprocess.Popen(cmd_args)
35    return subproc.communicate()
36
37def make(args, pipe=True):
38    return proc([make_cmd] + args, pipe)
39
40def git(args, pipe=True):
41    return proc([git_cmd] + args, pipe)
42
43def get_git_tags():
44    stdout, stderr = git(['tag', '-l', 'r[0-9][0-9][0-9]'])
45    tags = stdout.decode('utf-8').split()
46    stdout, stderr = git(['tag', '-l', 'v[1-9].[0-9].[0-9]'])
47    tags += stdout.decode('utf-8').split()
48    return tags
49
50# https://stackoverflow.com/a/19711609/2132223
51def sha1_of_file(filepath):
52    with open(filepath, 'rb') as f:
53        return hashlib.sha1(f.read()).hexdigest()
54
55if __name__ == '__main__':
56    error_code = 0
57    base_dir = os.getcwd() + '/..'           # /path/to/lz4
58    tmp_dir = base_dir + '/' + tmp_dir_name  # /path/to/lz4/tests/versionsTest
59    clone_dir = tmp_dir + '/' + 'lz4'        # /path/to/lz4/tests/versionsTest/lz4
60    programs_dir = base_dir + '/programs'    # /path/to/lz4/programs
61    os.makedirs(tmp_dir, exist_ok=True)
62
63    # since Travis clones limited depth, we should clone full repository
64    if not os.path.isdir(clone_dir):
65        git(['clone', repo_url, clone_dir])
66
67    shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
68
69    # Retrieve all release tags
70    print('Retrieve all release tags :')
71    os.chdir(clone_dir)
72    tags = [head] + get_git_tags()
73    print(tags);
74
75    # Build all release lz4c and lz4c32
76    for tag in tags:
77        os.chdir(base_dir)
78        dst_lz4c   = '{}/lz4c.{}'  .format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c.<TAG>
79        dst_lz4c32 = '{}/lz4c32.{}'.format(tmp_dir, tag) # /path/to/lz4/test/lz4test/lz4c32.<TAG>
80        if not os.path.isfile(dst_lz4c) or not os.path.isfile(dst_lz4c32) or tag == head:
81            if tag != head:
82                r_dir = '{}/{}'.format(tmp_dir, tag)  # /path/to/lz4/test/lz4test/<TAG>
83                os.makedirs(r_dir, exist_ok=True)
84                os.chdir(clone_dir)
85                git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
86                os.chdir(r_dir + '/programs')  # /path/to/lz4/lz4test/<TAG>/programs
87            else:
88                os.chdir(programs_dir)
89            make(['clean', 'lz4c'], False)
90            shutil.copy2('lz4c',   dst_lz4c)
91            make(['clean', 'lz4c32'], False)
92            shutil.copy2('lz4c32', dst_lz4c32)
93
94    # Compress test.dat by all released lz4c and lz4c32
95    print('Compress test.dat by all released lz4c and lz4c32')
96    os.chdir(tmp_dir)
97    for lz4 in glob.glob("*.lz4"):
98        os.remove(lz4)
99    for tag in tags:
100        proc(['./lz4c.'   + tag, '-1fz', test_dat, test_dat + '_1_64_' + tag + '.lz4'])
101        proc(['./lz4c.'   + tag, '-9fz', test_dat, test_dat + '_9_64_' + tag + '.lz4'])
102        proc(['./lz4c32.' + tag, '-1fz', test_dat, test_dat + '_1_32_' + tag + '.lz4'])
103        proc(['./lz4c32.' + tag, '-9fz', test_dat, test_dat + '_9_32_' + tag + '.lz4'])
104
105    print('Full list of compressed files')
106    lz4s = sorted(glob.glob('*.lz4'))
107    for lz4 in lz4s:
108        print(lz4 + ' : ' + repr(os.path.getsize(lz4)))
109
110    # Remove duplicated .lz4 files
111    print('')
112    print('Duplicated files')
113    lz4s = sorted(glob.glob('*.lz4'))
114    for i, lz4 in enumerate(lz4s):
115        if not os.path.isfile(lz4):
116            continue
117        for j in range(i+1, len(lz4s)):
118            lz4t = lz4s[j]
119            if not os.path.isfile(lz4t):
120                continue
121            if filecmp.cmp(lz4, lz4t):
122                os.remove(lz4t)
123                print('{} == {}'.format(lz4, lz4t))
124
125    print('Enumerate only different compressed files')
126    lz4s = sorted(glob.glob('*.lz4'))
127    for lz4 in lz4s:
128        print(lz4 + ' : ' + repr(os.path.getsize(lz4)) + ', ' + sha1_of_file(lz4))
129
130    # Decompress remained .lz4 files by all released lz4c and lz4c32
131    print('Decompression tests and verifications')
132    lz4s = sorted(glob.glob('*.lz4'))
133    for dec in glob.glob("*.dec"):
134        os.remove(dec)
135    for lz4 in lz4s:
136        print(lz4, end=" ")
137        for tag in tags:
138            print(tag, end=" ")
139            proc(['./lz4c.'   + tag, '-df', lz4, lz4 + '_d64_' + tag + '.dec'])
140            proc(['./lz4c32.' + tag, '-df', lz4, lz4 + '_d32_' + tag + '.dec'])
141        print(' OK')   # well, here, decompression has worked; but file is not yet verified
142
143    # Compare all '.dec' files with test_dat
144    decs = glob.glob('*.dec')
145    for dec in decs:
146        if not filecmp.cmp(dec, test_dat):
147            print('ERR : ' + dec)
148            error_code = 1
149        else:
150            print('OK  : ' + dec)
151            os.remove(dec)
152
153    if error_code != 0:
154        print('ERROR')
155
156    sys.exit(error_code)
157