comparison: tests/compare.py
tests/compare.py
- branch
- trunk
- changeset 17
- 481a2b4be471
- parent 15
- 28dbd52a8bb8
- child 19
- bb6e9f4af1a5
equal
deleted
inserted
replaced
| 16:29794d4d41aa | 17:481a2b4be471 |
|---|---|
| 16 # You should have received a copy of the GNU General Public License along | 16 # You should have received a copy of the GNU General Public License along |
| 17 # with this program; if not, write to the Free Software Foundation, Inc., | 17 # with this program; if not, write to the Free Software Foundation, Inc., |
| 18 # 51 Franklin Street, 5th Floor, Boston, MA, 02111. | 18 # 51 Franklin Street, 5th Floor, Boston, MA, 02111. |
| 19 | 19 |
| 20 import os | 20 import os |
| 21 import re | |
| 21 import subprocess | 22 import subprocess |
| 22 import syck | 23 import syck |
| 23 import sys | 24 import sys |
| 25 import tempfile | |
| 24 | 26 |
| 25 from sets import Set as set | 27 from sets import Set as set |
| 26 | 28 |
| 27 if os.path.exists('scripts/x') and os.path.exists('tests'): | 29 if os.path.exists('scripts/x') and os.path.exists('tests'): |
| 28 os.chdir('tests') | 30 os.chdir('tests') |
| 38 TESTSCRIPT_NAME = 'testscript.sh' | 40 TESTSCRIPT_NAME = 'testscript.sh' |
| 39 SCRIPT_PROLOGUE = """#!/bin/sh | 41 SCRIPT_PROLOGUE = """#!/bin/sh |
| 40 set -e | 42 set -e |
| 41 """ | 43 """ |
| 42 | 44 |
| 45 output_buffer = tempfile.TemporaryFile() | |
| 46 | |
| 43 class ExtractorTestError(Exception): | 47 class ExtractorTestError(Exception): |
| 44 pass | 48 pass |
| 45 | 49 |
| 46 | 50 |
| 47 class ExtractorTest(object): | 51 class ExtractorTest(object): |
| 48 def __init__(self, **kwargs): | 52 def __init__(self, **kwargs): |
| 49 for key in ('name', 'filename', 'baseline'): | 53 for key in ('name',): |
| 50 setattr(self, key, kwargs[key]) | 54 setattr(self, key, kwargs[key]) |
| 51 for key in ('directory', 'prerun', 'posttest'): | 55 for key in ('directory', 'prerun', 'posttest', 'baseline', 'error', |
| 56 'grep', 'antigrep'): | |
| 52 setattr(self, key, kwargs.get(key, None)) | 57 setattr(self, key, kwargs.get(key, None)) |
| 53 for key in ('options',): | 58 for key in ('options', 'filenames'): |
| 54 setattr(self, key, kwargs.get(key, '').split()) | 59 setattr(self, key, kwargs.get(key, '').split()) |
| 55 | 60 |
| 56 def get_results(self, commands): | 61 def get_results(self, commands): |
| 57 status = subprocess.call(commands) | 62 print >>output_buffer, "Output from %s:" % (' '.join(commands),) |
| 58 if status != 0: | 63 output_buffer.flush() |
| 59 return None | 64 status = subprocess.call(commands, stdout=output_buffer, |
| 60 process = subprocess.Popen(['find'], stdout=subprocess.PIPE) | 65 stderr=output_buffer) |
| 66 process = subprocess.Popen(['find', '!', '-name', TESTSCRIPT_NAME], | |
| 67 stdout=subprocess.PIPE) | |
| 61 process.wait() | 68 process.wait() |
| 62 output = process.stdout.read(-1) | 69 output = process.stdout.read(-1) |
| 63 process.stdout.close() | 70 process.stdout.close() |
| 64 return set(output.split('\n')) | 71 return status, set(output.split('\n')) |
| 65 | 72 |
| 66 def write_script(self, commands): | 73 def write_script(self, commands): |
| 67 script = open(TESTSCRIPT_NAME, 'w') | 74 script = open(TESTSCRIPT_NAME, 'w') |
| 68 script.write("%s%s\n" % (SCRIPT_PROLOGUE, commands)) | 75 script.write("%s%s\n" % (SCRIPT_PROLOGUE, commands)) |
| 69 script.close() | 76 script.close() |
| 70 subprocess.call(['chmod', 'u+w', TESTSCRIPT_NAME]) | 77 subprocess.call(['chmod', 'u+w', TESTSCRIPT_NAME]) |
| 71 | 78 |
| 72 def get_shell_results(self): | 79 def get_shell_results(self): |
| 73 self.write_script(self.baseline) | 80 self.write_script(self.baseline) |
| 74 return self.get_results(['sh', TESTSCRIPT_NAME, self.filename]) | 81 return self.get_results(['sh', TESTSCRIPT_NAME] + self.filenames) |
| 75 | 82 |
| 76 def get_extractor_results(self): | 83 def get_extractor_results(self): |
| 77 if self.prerun: | 84 if self.prerun: |
| 78 self.write_script(self.prerun) | 85 self.write_script(self.prerun) |
| 79 subprocess.call(['sh', TESTSCRIPT_NAME]) | 86 subprocess.call(['sh', TESTSCRIPT_NAME]) |
| 80 return self.get_results([X_SCRIPT] + self.options + [self.filename]) | 87 return self.get_results([X_SCRIPT] + self.options + self.filenames) |
| 81 | 88 |
| 82 def get_posttest_result(self): | 89 def get_posttest_result(self): |
| 83 if not self.posttest: | 90 if not self.posttest: |
| 84 return 0 | 91 return 0 |
| 85 self.write_script(self.posttest) | 92 self.write_script(self.posttest) |
| 101 if status != 0: | 108 if status != 0: |
| 102 raise ExtractorTestError("cleanup exited with status code %s" % | 109 raise ExtractorTestError("cleanup exited with status code %s" % |
| 103 (status,)) | 110 (status,)) |
| 104 | 111 |
| 105 def show_status(self, status, message=None): | 112 def show_status(self, status, message=None): |
| 113 raw_status = status.lower() | |
| 114 if raw_status != 'passed': | |
| 115 output_buffer.seek(0, 0) | |
| 116 sys.stdout.write(output_buffer.read(-1)) | |
| 106 if message is None: | 117 if message is None: |
| 107 last_part = '' | 118 last_part = '' |
| 108 else: | 119 else: |
| 109 last_part = ': ' + str(message) | 120 last_part = ': %s' % (message,) |
| 110 print "%7s: %s%s" % (status, self.name, last_part) | 121 print "%7s: %s%s" % (status, self.name, last_part) |
| 111 return status.lower() | 122 return raw_status |
| 112 | 123 |
| 113 def compare_results(self): | 124 def compare_results(self, actual): |
| 114 self.clean() | |
| 115 expected = self.get_shell_results() | |
| 116 self.clean() | |
| 117 actual = self.get_extractor_results() | |
| 118 posttest_result = self.get_posttest_result() | 125 posttest_result = self.get_posttest_result() |
| 119 self.clean() | 126 self.clean() |
| 120 if expected is None: | 127 status, expected = self.get_shell_results() |
| 121 raise ExtractorTestError("could not get baseline results") | 128 self.clean() |
| 122 elif actual is None: | 129 if expected != actual: |
| 123 raise ExtractorTestError("could not get extractor results") | 130 print >>output_buffer, "Only in baseline results:" |
| 124 elif expected != actual: | 131 print >>output_buffer, '\n'.join(expected.difference(actual)) |
| 125 result = self.show_status('FAILED') | 132 print >>output_buffer, "Only in actual results:" |
| 126 print "Only in baseline results:" | 133 print >>output_buffer, '\n'.join(actual.difference(expected)) |
| 127 print '\n'.join(expected.difference(actual)) | 134 return self.show_status('FAILED') |
| 128 print "Only in actual results:" | |
| 129 print '\n'.join(actual.difference(expected)) | |
| 130 elif posttest_result != 0: | 135 elif posttest_result != 0: |
| 131 result = self.show_status('FAILED') | 136 print >>output_buffer, "Posttest gave status code", posttest_result |
| 132 print "Posttest returned status code", posttest_result | 137 return self.show_status('FAILED') |
| 133 else: | 138 return self.show_status('Passed') |
| 134 result = self.show_status('Passed') | |
| 135 return result | |
| 136 | 139 |
| 140 def have_error_mismatch(self, status): | |
| 141 if self.error and (status == 0): | |
| 142 return "x did not return expected error" | |
| 143 elif (not self.error) and (status != 0): | |
| 144 return "x returned error code %s" % (status,) | |
| 145 return None | |
| 146 | |
| 147 def grep_output(self): | |
| 148 output_buffer.seek(0, 0) | |
| 149 output_buffer.readline() | |
| 150 output = output_buffer.read(-1) | |
| 151 if self.grep and (not re.search(self.grep, output)): | |
| 152 return "output did not match %s" % (self.grep) | |
| 153 elif self.antigrep and re.search(self.antigrep, output): | |
| 154 return "output matched antigrep %s" % (self.antigrep) | |
| 155 return None | |
| 156 | |
| 157 def check_results(self): | |
| 158 output_buffer.seek(0, 0) | |
| 159 output_buffer.truncate() | |
| 160 self.clean() | |
| 161 status, actual = self.get_extractor_results() | |
| 162 problem = self.have_error_mismatch(status) or self.grep_output() | |
| 163 if problem: | |
| 164 return self.show_status('FAILED', problem) | |
| 165 return self.compare_results(actual) | |
| 166 | |
| 137 def run(self): | 167 def run(self): |
| 138 if self.directory: | 168 if self.directory: |
| 139 os.mkdir(self.directory) | 169 os.mkdir(self.directory) |
| 140 os.chdir(self.directory) | 170 os.chdir(self.directory) |
| 141 try: | 171 try: |
| 142 result = self.compare_results() | 172 result = self.check_results() |
| 143 except ExtractorTestError, error: | 173 except ExtractorTestError, error: |
| 144 result = self.show_status('ERROR', error) | 174 result = self.show_status('ERROR', error) |
| 145 if self.directory: | 175 if self.directory: |
| 146 os.chdir(ROOT_DIR) | 176 os.chdir(ROOT_DIR) |
| 177 subprocess.call(['chmod', '-R', '700', self.directory]) | |
| 147 subprocess.call(['rm', '-rf', self.directory]) | 178 subprocess.call(['rm', '-rf', self.directory]) |
| 148 return result | 179 return result |
| 149 | 180 |
| 150 | 181 |
| 151 test_db = open('tests.yml') | 182 test_db = open('tests.yml') |
| 152 test_data = syck.load(test_db.read(-1)) | 183 test_data = syck.load(test_db.read(-1)) |
| 153 test_db.close() | 184 test_db.close() |
| 154 tests = [ExtractorTest(**data) for data in test_data] | 185 tests = [ExtractorTest(**data) for data in test_data] |
| 155 for original_data in test_data: | 186 for original_data in test_data: |
| 156 if original_data.has_key('directory'): | 187 if (original_data.has_key('directory') or |
| 188 (not original_data.has_key('baseline'))): | |
| 157 continue | 189 continue |
| 158 data = original_data.copy() | 190 data = original_data.copy() |
| 159 data['name'] += ' in ..' | 191 data['name'] += ' in ..' |
| 160 data['directory'] = 'inside-dir' | 192 data['directory'] = 'inside-dir' |
| 161 data['filename'] = os.path.join('..', data['filename']) | 193 data['filenames'] = ' '.join(['../%s' % filename for filename in |
| 194 data.get('filenames', '').split()]) | |
| 162 tests.append(ExtractorTest(**data)) | 195 tests.append(ExtractorTest(**data)) |
| 163 results = [test.run() for test in tests] | 196 results = [test.run() for test in tests] |
| 164 counts = {} | 197 counts = {} |
| 165 for outcome in OUTCOMES: | 198 for outcome in OUTCOMES: |
| 166 counts[outcome] = 0 | 199 counts[outcome] = 0 |
| 167 for result in results: | 200 for result in results: |
| 168 counts[result] += 1 | 201 counts[result] += 1 |
| 169 print " Totals:", ', '.join(["%s %s" % (counts[key], key) for key in OUTCOMES]) | 202 print " Totals:", ', '.join(["%s %s" % (counts[key], key) for key in OUTCOMES]) |
| 203 output_buffer.close() |
