comparison: tests/compare.py
tests/compare.py
- branch
- trunk
- changeset 14
- 6f9e1bb59719
- parent 10
- f0acfe12a0e2
- child 15
- 28dbd52a8bb8
equal
deleted
inserted
replaced
| 13:0a3ef1b9f6d4 | 14:6f9e1bb59719 |
|---|---|
| 17 # with this program; if not, write to the Free Software Foundation, Inc., | 17 # with this program; if not, write to the Free Software Foundation, Inc., |
| 18 # 51 Franklin Street, 5th Floor, Boston, MA, 02111. | 18 # 51 Franklin Street, 5th Floor, Boston, MA, 02111. |
| 19 | 19 |
| 20 import os | 20 import os |
| 21 import subprocess | 21 import subprocess |
| 22 import syck | |
| 22 import sys | 23 import sys |
| 23 | 24 |
| 24 from sets import Set as set | 25 from sets import Set as set |
| 25 | |
| 26 TESTSCRIPT_NAME = 'testscript.sh' | |
| 27 SCRIPT_PROLOGUE = """#!/bin/sh | |
| 28 set -e | |
| 29 """ | |
| 30 | |
| 31 tests = {'test-1.23.tar': ([], ['tar -xf $1'], []), | |
| 32 'test-1.23.tar.gz': ([], ['tar -xzf $1'], []), | |
| 33 'test-1.23.tar.bz2': ([], ['mkdir test-1.23', | |
| 34 'cd test-1.23', | |
| 35 'tar -jxf ../$1'], []), | |
| 36 'test-1.23.zip': ([], ['mkdir test-1.23', | |
| 37 'cd test-1.23', | |
| 38 'unzip -q ../$1'], []), | |
| 39 'test-1.23.cpio': ([], ['cpio -i --make-directories \ | |
| 40 <$1 2>/dev/null'], []), | |
| 41 'test-1.23_all.deb': ([], ['TD=$PWD', | |
| 42 'mkdir test-1.23', | |
| 43 'cd /tmp', | |
| 44 'ar x $TD/$1 data.tar.gz', | |
| 45 'cd $TD/test-1.23', | |
| 46 'tar -zxf /tmp/data.tar.gz', | |
| 47 'rm /tmp/data.tar.gz'], []), | |
| 48 'test-recursive-badperms.tar.bz2': ( | |
| 49 ['-r'], | |
| 50 ['mkdir test-recursive-badperms', | |
| 51 'cd test-recursive-badperms', | |
| 52 'tar -jxf ../$1', | |
| 53 'mkdir test-badperms', | |
| 54 'cd test-badperms', | |
| 55 'tar -xf ../test-badperms.tar', | |
| 56 'chmod 755 testdir'], | |
| 57 ['if [ "x`cat test-recursive-badperms/test-badperms/testdir/testfile`" = \ | |
| 58 "xhey" ]', | |
| 59 'then exit 0; else exit 1; fi'] | |
| 60 )} | |
| 61 | 26 |
| 62 if os.path.exists('scripts/x') and os.path.exists('tests'): | 27 if os.path.exists('scripts/x') and os.path.exists('tests'): |
| 63 os.chdir('tests') | 28 os.chdir('tests') |
| 64 elif os.path.exists('../scripts/x') and os.path.exists('../tests'): | 29 elif os.path.exists('../scripts/x') and os.path.exists('../tests'): |
| 65 pass | 30 pass |
| 66 else: | 31 else: |
| 67 print "ERROR: Can't run tests in this directory!" | 32 print "ERROR: Can't run tests in this directory!" |
| 68 sys.exit(2) | 33 sys.exit(2) |
| 69 | 34 |
| 35 X_SCRIPT = os.path.realpath('../scripts/x') | |
| 36 ROOT_DIR = os.path.realpath(os.curdir) | |
| 37 OUTCOMES = ['error', 'failed', 'passed'] | |
| 38 TESTSCRIPT_NAME = 'testscript.sh' | |
| 39 SCRIPT_PROLOGUE = """#!/bin/sh | |
| 40 set -e | |
| 41 """ | |
| 42 | |
| 70 class ExtractorTestError(Exception): | 43 class ExtractorTestError(Exception): |
| 71 pass | 44 pass |
| 72 | 45 |
| 73 | 46 |
| 74 class ExtractorTest(object): | 47 class ExtractorTest(object): |
| 75 def __init__(self, directory, archive_filename, info): | 48 def __init__(self, **kwargs): |
| 76 self.directory = directory | 49 for key in ('name', 'filename', 'baseline'): |
| 77 self.archive_filename = os.path.join(directory, archive_filename) | 50 setattr(self, key, kwargs[key]) |
| 78 self.arguments, self.shell_commands, self.shell_test = info | 51 for key in ('directory', 'prerun', 'posttest'): |
| 52 setattr(self, key, kwargs.get(key, None)) | |
| 53 for key in ('options',): | |
| 54 setattr(self, key, kwargs.get(key, '').split()) | |
| 79 | 55 |
| 80 def get_results(self, commands): | 56 def get_results(self, commands): |
| 81 status = subprocess.call(commands) | 57 status = subprocess.call(commands) |
| 82 if status != 0: | 58 if status != 0: |
| 83 return None | 59 return None |
| 87 process.stdout.close() | 63 process.stdout.close() |
| 88 return set(output.split('\n')) | 64 return set(output.split('\n')) |
| 89 | 65 |
| 90 def write_script(self, commands): | 66 def write_script(self, commands): |
| 91 script = open(TESTSCRIPT_NAME, 'w') | 67 script = open(TESTSCRIPT_NAME, 'w') |
| 92 script.write("%s%s\n" % (SCRIPT_PROLOGUE, '\n'.join(commands))) | 68 script.write("%s%s\n" % (SCRIPT_PROLOGUE, commands)) |
| 93 script.close() | 69 script.close() |
| 94 subprocess.call(['chmod', 'u+w', TESTSCRIPT_NAME]) | 70 subprocess.call(['chmod', 'u+w', TESTSCRIPT_NAME]) |
| 95 | 71 |
| 96 def get_shell_results(self): | 72 def get_shell_results(self): |
| 97 self.write_script(self.shell_commands) | 73 self.write_script(self.baseline) |
| 98 return self.get_results(['sh', TESTSCRIPT_NAME, self.archive_filename]) | 74 return self.get_results(['sh', TESTSCRIPT_NAME, self.filename]) |
| 99 | 75 |
| 100 def get_extractor_results(self): | 76 def get_extractor_results(self): |
| 101 script = os.path.join(self.directory, '../scripts/x') | 77 if self.prerun: |
| 102 return self.get_results([script] + self.arguments + | 78 self.write_script(self.prerun) |
| 103 [self.archive_filename]) | 79 subprocess.call(['sh', TESTSCRIPT_NAME]) |
| 80 return self.get_results([X_SCRIPT] + self.options + [self.filename]) | |
| 104 | 81 |
| 105 def get_posttest_result(self): | 82 def get_posttest_result(self): |
| 106 if not self.shell_test: | 83 if not self.posttest: |
| 107 return 0 | 84 return 0 |
| 108 self.write_script(self.shell_test) | 85 self.write_script(self.posttest) |
| 109 return subprocess.call(['sh', TESTSCRIPT_NAME]) | 86 return subprocess.call(['sh', TESTSCRIPT_NAME]) |
| 110 | 87 |
| 111 def clean(self): | 88 def clean(self): |
| 112 status = subprocess.call(['find', '-mindepth', '1', '-maxdepth', '1', | 89 if self.directory: |
| 113 '-type', 'd', | 90 target = os.path.join(ROOT_DIR, self.directory) |
| 114 '!', '-name', 'CVS', '!', '-name', '.svn', | 91 extra_options = ['!', '-name', TESTSCRIPT_NAME] |
| 115 '-exec', 'rm', '-rf', '{}', ';']) | 92 else: |
| 93 target = ROOT_DIR | |
| 94 extra_options = ['-type', 'd', | |
| 95 '!', '-name', 'CVS', | |
| 96 '!', '-name', '.svn'] | |
| 97 status = subprocess.call(['find', target, | |
| 98 '-mindepth', '1', '-maxdepth', '1'] + | |
| 99 extra_options + | |
| 100 ['-exec', 'rm', '-rf', '{}', ';']) | |
| 116 if status != 0: | 101 if status != 0: |
| 117 raise ExtractorTestError("cleanup exited with status code %s" % | 102 raise ExtractorTestError("cleanup exited with status code %s" % |
| 118 (status,)) | 103 (status,)) |
| 119 | 104 |
| 120 def run(self): | 105 def show_status(self, status, message=None): |
| 106 if message is None: | |
| 107 last_part = '' | |
| 108 else: | |
| 109 last_part = ': ' + message | |
| 110 print "%7s: %s%s" % (status, self.name, last_part) | |
| 111 return status.lower() | |
| 112 | |
| 113 def compare_results(self): | |
| 121 self.clean() | 114 self.clean() |
| 122 expected = self.get_shell_results() | 115 expected = self.get_shell_results() |
| 123 self.clean() | 116 self.clean() |
| 124 actual = self.get_extractor_results() | 117 actual = self.get_extractor_results() |
| 125 posttest_result = self.get_posttest_result() | 118 posttest_result = self.get_posttest_result() |
| 127 if expected is None: | 120 if expected is None: |
| 128 raise ExtractorTestError("could not get baseline results") | 121 raise ExtractorTestError("could not get baseline results") |
| 129 elif actual is None: | 122 elif actual is None: |
| 130 raise ExtractorTestError("could not get extractor results") | 123 raise ExtractorTestError("could not get extractor results") |
| 131 elif expected != actual: | 124 elif expected != actual: |
| 132 print "FAILED:", self.archive_filename | 125 result = self.show_status('FAILED') |
| 133 print "Only in baseline results:" | 126 print "Only in baseline results:" |
| 134 print '\n'.join(expected.difference(actual)) | 127 print '\n'.join(expected.difference(actual)) |
| 135 print "Only in actual results:" | 128 print "Only in actual results:" |
| 136 print '\n'.join(actual.difference(expected)) | 129 print '\n'.join(actual.difference(expected)) |
| 137 return False | |
| 138 elif posttest_result != 0: | 130 elif posttest_result != 0: |
| 139 print "FAILED:", self.archive_filename | 131 result = self.show_status('FAILED') |
| 140 print "Posttest returned status code", posttest_result | 132 print "Posttest returned status code", posttest_result |
| 141 print actual | |
| 142 return False | |
| 143 else: | 133 else: |
| 144 print "Passed:", self.archive_filename | 134 result = self.show_status('Passed') |
| 145 return True | 135 return result |
| 136 | |
| 137 def run(self): | |
| 138 if self.directory: | |
| 139 os.mkdir(self.directory) | |
| 140 os.chdir(self.directory) | |
| 141 try: | |
| 142 result = self.compare_results() | |
| 143 except ExtractorTestError, error: | |
| 144 result = self.show_status('ERROR', error) | |
| 145 if self.directory: | |
| 146 os.chdir(ROOT_DIR) | |
| 147 subprocess.call(['rm', '-rf', self.directory]) | |
| 148 return result | |
| 146 | 149 |
| 147 | 150 |
| 148 def run_tests(directory, testnames): | 151 test_db = open('tests.yml') |
| 149 successes = 0 | 152 test_data = syck.load(test_db.read(-1)) |
| 150 failures = 0 | 153 test_db.close() |
| 151 for testname in testnames: | 154 tests = [ExtractorTest(**data) for data in test_data] |
| 152 test = ExtractorTest(directory, testname, tests[testname]) | 155 for original_data in test_data: |
| 153 if test.run(): | 156 if original_data.has_key('directory'): |
| 154 successes += 1 | 157 continue |
| 155 else: | 158 data = original_data.copy() |
| 156 failures += 1 | 159 data['name'] += ' in ..' |
| 157 return successes, failures | 160 data['directory'] = 'inside-dir' |
| 158 | 161 data['filename'] = os.path.join('..', data['filename']) |
| 159 results = [] | 162 tests.append(ExtractorTest(**data)) |
| 160 testnames = tests.keys() | 163 results = [test.run() for test in tests] |
| 161 testnames.sort() | 164 counts = {} |
| 162 results.append(run_tests('.', testnames)) | 165 for outcome in OUTCOMES: |
| 163 os.mkdir('inside-dir') | 166 counts[outcome] = 0 |
| 164 os.chdir('inside-dir') | 167 for result in results: |
| 165 results.append(run_tests('..', testnames)) | 168 counts[result] += 1 |
| 166 os.chdir('..') | 169 print " Totals:", ', '.join(["%s %s" % (counts[key], key) for key in OUTCOMES]) |
| 167 subprocess.call(['rm', '-rf', 'inside-dir']) | |
| 168 print "Totals: %s successes, %s failures" % \ | |
| 169 tuple([sum(total) for total in zip(*results)]) |
