1#!/usr/bin/env python3 2"""Test zstd interoperability between versions""" 3 4# ################################################################ 5# Copyright (c) Yann Collet, Facebook, Inc. 6# All rights reserved. 7# 8# This source code is licensed under both the BSD-style license (found in the 9# LICENSE file in the root directory of this source tree) and the GPLv2 (found 10# in the COPYING file in the root directory of this source tree). 11# You may select, at your option, one of the above-listed licenses. 12# ################################################################ 13 14import filecmp 15import glob 16import hashlib 17import os 18import shutil 19import sys 20import subprocess 21from subprocess import Popen, PIPE 22 23repo_url = 'https://github.com/facebook/zstd.git' 24tmp_dir_name = 'tests/versionsTest' 25make_cmd = 'make' 26make_args = ['-j','CFLAGS=-O1'] 27git_cmd = 'git' 28test_dat_src = 'README.md' 29test_dat = 'test_dat' 30head = 'vdevel' 31dict_source = 'dict_source' 32dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c ' 33dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h' 34 35 36def execute(command, print_output=False, print_error=True, param_shell=False): 37 popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell) 38 stdout_lines, stderr_lines = popen.communicate() 39 stderr_lines = stderr_lines.decode("utf-8") 40 stdout_lines = stdout_lines.decode("utf-8") 41 if print_output: 42 print(stdout_lines) 43 print(stderr_lines) 44 if popen.returncode is not None and popen.returncode != 0: 45 if not print_output and print_error: 46 print(stderr_lines) 47 return popen.returncode 48 49 50def proc(cmd_args, pipe=True, dummy=False): 51 if dummy: 52 return 53 if pipe: 54 subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE) 55 else: 56 subproc = Popen(cmd_args) 57 return subproc.communicate() 58 59 60def make(targets, pipe=True): 61 cmd = [make_cmd] + make_args + targets 62 cmd_str = str(cmd) 63 print('compilation command : ' + cmd_str) 64 return proc(cmd, pipe) 65 66 67def git(args, pipe=True): 68 return proc([git_cmd] + args, pipe) 69 70 71def get_git_tags(): 72 stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]']) 73 tags = stdout.decode('utf-8').split() 74 return tags 75 76 77def create_dict(tag, dict_source_path): 78 dict_name = 'dict.' + tag 79 if not os.path.isfile(dict_name): 80 cFiles = glob.glob(dict_source_path + "/*.c") 81 hFiles = glob.glob(dict_source_path + "/*.h") 82 if tag == 'v0.5.0': 83 result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) 84 else: 85 result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) 86 if result == 0: 87 print(dict_name + ' created') 88 else: 89 print('ERROR: creating of ' + dict_name + ' failed') 90 else: 91 print(dict_name + ' already exists') 92 93 94def dict_compress_sample(tag, sample): 95 dict_name = 'dict.' + tag 96 DEVNULL = open(os.devnull, 'wb') 97 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0: 98 os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst') 99 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0: 100 os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst') 101 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0: 102 os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst') 103 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0: 104 os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst') 105 if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0: 106 os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst') 107 # zstdFiles = glob.glob("*.zst*") 108 # print(zstdFiles) 109 print(tag + " : dict compression completed") 110 111 112def compress_sample(tag, sample): 113 DEVNULL = open(os.devnull, 'wb') 114 if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: 115 os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') 116 if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: 117 os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst') 118 if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0: 119 os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst') 120 if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0: 121 os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst') 122 if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0: 123 os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst') 124 # zstdFiles = glob.glob("*.zst*") 125 # print(zstdFiles) 126 print(tag + " : compression completed") 127 128 129# http://stackoverflow.com/a/19711609/2132223 130def sha1_of_file(filepath): 131 with open(filepath, 'rb') as f: 132 return hashlib.sha1(f.read()).hexdigest() 133 134 135def remove_duplicates(): 136 list_of_zst = sorted(glob.glob('*.zst')) 137 for i, ref_zst in enumerate(list_of_zst): 138 if not os.path.isfile(ref_zst): 139 continue 140 for j in range(i + 1, len(list_of_zst)): 141 compared_zst = list_of_zst[j] 142 if not os.path.isfile(compared_zst): 143 continue 144 if filecmp.cmp(ref_zst, compared_zst): 145 os.remove(compared_zst) 146 print('duplicated : {} == {}'.format(ref_zst, compared_zst)) 147 148 149def decompress_zst(tag): 150 dec_error = 0 151 list_zst = sorted(glob.glob('*_nodict.zst')) 152 for file_zst in list_zst: 153 print(file_zst, end=' ') 154 print(tag, end=' ') 155 file_dec = file_zst + '_d64_' + tag + '.dec' 156 if tag <= 'v0.5.0': 157 params = ['./zstd.' + tag, '-df', file_zst, file_dec] 158 else: 159 params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] 160 if execute(params) == 0: 161 if not filecmp.cmp(file_dec, test_dat): 162 print('ERR !! ') 163 dec_error = 1 164 else: 165 print('OK ') 166 else: 167 print('command does not work') 168 dec_error = 1 169 return dec_error 170 171 172def decompress_dict(tag): 173 dec_error = 0 174 list_zst = sorted(glob.glob('*_dictio.zst')) 175 for file_zst in list_zst: 176 dict_tag = file_zst[0:len(file_zst)-11] # remove "_dictio.zst" 177 if head in dict_tag: # find vdevel 178 dict_tag = head 179 else: 180 dict_tag = dict_tag[dict_tag.rfind('v'):] 181 if tag == 'v0.6.0' and dict_tag < 'v0.6.0': 182 continue 183 dict_name = 'dict.' + dict_tag 184 print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ') 185 file_dec = file_zst + '_d64_' + tag + '.dec' 186 if tag <= 'v0.5.0': 187 params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] 188 else: 189 params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] 190 if execute(params) == 0: 191 if not filecmp.cmp(file_dec, test_dat): 192 print('ERR !! ') 193 dec_error = 1 194 else: 195 print('OK ') 196 else: 197 print('command does not work') 198 dec_error = 1 199 return dec_error 200 201 202if __name__ == '__main__': 203 error_code = 0 204 base_dir = os.getcwd() + '/..' # /path/to/zstd 205 tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest 206 clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd 207 dict_source_path = tmp_dir + '/' + dict_source # /path/to/zstd/tests/versionsTest/dict_source 208 programs_dir = base_dir + '/programs' # /path/to/zstd/programs 209 os.makedirs(tmp_dir, exist_ok=True) 210 211 # since Travis clones limited depth, we should clone full repository 212 if not os.path.isdir(clone_dir): 213 git(['clone', repo_url, clone_dir]) 214 215 shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat) 216 217 # Retrieve all release tags 218 print('Retrieve all release tags :') 219 os.chdir(clone_dir) 220 alltags = get_git_tags() + [head] 221 tags = [t for t in alltags if t >= 'v0.5.0'] 222 print(tags) 223 224 # Build all release zstd 225 for tag in tags: 226 os.chdir(base_dir) 227 dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd.<TAG> 228 if not os.path.isfile(dst_zstd) or tag == head: 229 if tag != head: 230 print('-----------------------------------------------') 231 print('compiling ' + tag) 232 print('-----------------------------------------------') 233 r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/<TAG> 234 os.makedirs(r_dir, exist_ok=True) 235 os.chdir(clone_dir) 236 git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) 237 if tag == 'v0.5.0': 238 os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder 239 make(['clean'], False) # separate 'clean' target to allow parallel build 240 make(['dictBuilder'], False) 241 shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag)) 242 os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs 243 make(['clean'], False) # separate 'clean' target to allow parallel build 244 make(['zstd'], False) 245 else: 246 os.chdir(programs_dir) 247 print('-----------------------------------------------') 248 print('compiling head') 249 print('-----------------------------------------------') 250 make(['zstd'], False) 251 shutil.copy2('zstd', dst_zstd) 252 253 # remove any remaining *.zst and *.dec from previous test 254 os.chdir(tmp_dir) 255 for compressed in glob.glob("*.zst"): 256 os.remove(compressed) 257 for dec in glob.glob("*.dec"): 258 os.remove(dec) 259 260 # copy *.c and *.h to a temporary directory ("dict_source") 261 if not os.path.isdir(dict_source_path): 262 os.mkdir(dict_source_path) 263 print('cp ' + dict_files + ' ' + dict_source_path) 264 execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True) 265 266 print('-----------------------------------------------') 267 print('Compress test.dat by all released zstd') 268 print('-----------------------------------------------') 269 270 error_code = 0 271 for tag in tags: 272 print(tag) 273 if tag >= 'v0.5.0': 274 create_dict(tag, dict_source_path) 275 dict_compress_sample(tag, test_dat) 276 remove_duplicates() 277 error_code += decompress_dict(tag) 278 compress_sample(tag, test_dat) 279 remove_duplicates() 280 error_code += decompress_zst(tag) 281 282 print('') 283 print('Enumerate different compressed files') 284 zstds = sorted(glob.glob('*.zst')) 285 for zstd in zstds: 286 print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd)) 287 288 if error_code != 0: 289 print('====== ERROR !!! =======') 290 291 sys.exit(error_code) 292