# HG changeset patch # User brett # Date 1167611386 18000 # Node ID 481a2b4be471f165b82420c0194ce919ad662e50 # Parent 29794d4d41aa3e1712441903672283833b46fae1 [svn] Lots of tests for various boundary cases, and slightly better handling for some of them. diff -r 29794d4d41aa -r 481a2b4be471 scripts/x --- a/scripts/x Sun Dec 31 19:27:23 2006 -0500 +++ b/scripts/x Sun Dec 31 19:29:46 2006 -0500 @@ -70,12 +70,7 @@ def __init__(self, original_name): self.original_name = original_name - def is_free(self, filename=None): - if filename is None: - filename = self.original_name - return self._is_free(filename) - - def _is_free(self, filename): + def is_free(self, filename): return not os.path.exists(filename) def check(self): @@ -88,7 +83,7 @@ class DirectoryChecker(FilenameChecker): - def _is_free(self, filename): + def is_free(self, filename): try: os.mkdir(filename) except OSError, error: @@ -322,42 +317,17 @@ self.extractor = extractor self.contents = contents self.options = options - -## def extract(self): -## raise NotImplementedError -## checker = self.extractor.name_checker(self.extractor.basename()) -## if self.options.flat: -## self.target = '.' -## return self.do_extract('.') -## elif self.options.overwrite or checker.is_free(): -## self.target = self.extractor.basename() -## return self.overwrite() -## else: -## self.target = checker.check() -## return self.safe_extract() + self.target = None def extract(self): try: self.extractor.extract(self.target) - except ExtractorError, error: + except (ExtractorError, IOError, OSError), error: return str(error) def cleanup(self): - raise NotImplementedError - if self.options.flat: - self.cleanup_files() - else: - self.cleanup_directory() - - def cleanup_files(self): - for filename in self.extractor.get_filenames(): - stat_info = os.stat(filename) - perms = stat.S_IRUSR | stat.S_IWUSR - if stat.S_ISDIR(stat_info.st_mode): - perms |= stat.S_IXUSR - os.chmod(filename, stat_info.st_mode | perms) - - def cleanup_directory(self): + if self.target is None: + return command = 'find' status = subprocess.call(['find', self.target, '-type', 'd', '-exec', 'chmod', 'u+rwx', '{}', ';']) @@ -376,15 +346,23 @@ # Match . . tempdir + checked # Bomb . basename DirectoryChecked - class FlatHandler(BaseHandler): def can_handle(contents, options): return ((options.flat and (contents != COMPRESSED)) or (options.overwrite and (contents == MATCHING_DIRECTORY))) can_handle = staticmethod(can_handle) - target = '.' - cleanup = BaseHandler.cleanup_files + def __init__(self, extractor, contents, options): + BaseHandler.__init__(self, extractor, contents, options) + self.target = '.' + + def cleanup(self): + for filename in self.extractor.get_filenames(): + stat_info = os.stat(filename) + perms = stat.S_IRUSR | stat.S_IWUSR + if stat.S_ISDIR(stat_info.st_mode): + perms |= stat.S_IXUSR + os.chmod(filename, stat_info.st_mode | perms) class OverwriteHandler(BaseHandler): @@ -396,8 +374,6 @@ def __init__(self, extractor, contents, options): BaseHandler.__init__(self, extractor, contents, options) self.target = self.extractor.basename() - - cleanup = BaseHandler.cleanup_directory class MatchHandler(BaseHandler): @@ -407,7 +383,7 @@ def extract(self): basename = self.extractor.basename() - self.target = tempfile.mkdtemp() + self.target = tempfile.mkdtemp(dir='.') result = BaseHandler.extract(self) if result is None: tempdir = self.target @@ -417,8 +393,6 @@ os.rmdir(tempdir) return result - cleanup = BaseHandler.cleanup_directory - class EmptyHandler(object): def can_handle(contents, options): @@ -440,18 +414,7 @@ checker = self.extractor.name_checker(self.extractor.basename()) self.target = checker.check() - cleanup = BaseHandler.cleanup_directory - -## class BombHandler(BaseHandler): -## def safe_extract(self): -## return self.do_extract(self.target) - -## def overwrite(self): -## self.target = self.extractor.basename() -## return self.do_extract(self.target) - - extractor_map = {'application/x-tar': TarExtractor, 'application/zip': ZipExtractor, 'application/x-msdos-program': ZipExtractor, @@ -540,7 +503,10 @@ self.archives.setdefault(directory, []).append(basename) def report(self, function, *args): - error = function(*args) + try: + error = function(*args) + except (ExtractorError, IOError, OSError), exception: + error = str(exception) if error: self.logger.error("%s: %s", self.current_filename, error) return False @@ -552,7 +518,6 @@ for filename in filenames: os.chdir(self.current_directory) self.current_filename = filename - self.cleanup_actions = [] success = self.report(self.get_extractor) if success: for name in 'extract', 'cleanup': diff -r 29794d4d41aa -r 481a2b4be471 tests/compare.py --- a/tests/compare.py Sun Dec 31 19:27:23 2006 -0500 +++ b/tests/compare.py Sun Dec 31 19:29:46 2006 -0500 @@ -18,9 +18,11 @@ # 51 Franklin Street, 5th Floor, Boston, MA, 02111. import os +import re import subprocess import syck import sys +import tempfile from sets import Set as set @@ -40,28 +42,33 @@ set -e """ +output_buffer = tempfile.TemporaryFile() + class ExtractorTestError(Exception): pass class ExtractorTest(object): def __init__(self, **kwargs): - for key in ('name', 'filename', 'baseline'): + for key in ('name',): setattr(self, key, kwargs[key]) - for key in ('directory', 'prerun', 'posttest'): + for key in ('directory', 'prerun', 'posttest', 'baseline', 'error', + 'grep', 'antigrep'): setattr(self, key, kwargs.get(key, None)) - for key in ('options',): + for key in ('options', 'filenames'): setattr(self, key, kwargs.get(key, '').split()) def get_results(self, commands): - status = subprocess.call(commands) - if status != 0: - return None - process = subprocess.Popen(['find'], stdout=subprocess.PIPE) + print >>output_buffer, "Output from %s:" % (' '.join(commands),) + output_buffer.flush() + status = subprocess.call(commands, stdout=output_buffer, + stderr=output_buffer) + process = subprocess.Popen(['find', '!', '-name', TESTSCRIPT_NAME], + stdout=subprocess.PIPE) process.wait() output = process.stdout.read(-1) process.stdout.close() - return set(output.split('\n')) + return status, set(output.split('\n')) def write_script(self, commands): script = open(TESTSCRIPT_NAME, 'w') @@ -71,13 +78,13 @@ def get_shell_results(self): self.write_script(self.baseline) - return self.get_results(['sh', TESTSCRIPT_NAME, self.filename]) + return self.get_results(['sh', TESTSCRIPT_NAME] + self.filenames) def get_extractor_results(self): if self.prerun: self.write_script(self.prerun) subprocess.call(['sh', TESTSCRIPT_NAME]) - return self.get_results([X_SCRIPT] + self.options + [self.filename]) + return self.get_results([X_SCRIPT] + self.options + self.filenames) def get_posttest_result(self): if not self.posttest: @@ -103,47 +110,71 @@ (status,)) def show_status(self, status, message=None): + raw_status = status.lower() + if raw_status != 'passed': + output_buffer.seek(0, 0) + sys.stdout.write(output_buffer.read(-1)) if message is None: last_part = '' else: - last_part = ': ' + str(message) + last_part = ': %s' % (message,) print "%7s: %s%s" % (status, self.name, last_part) - return status.lower() + return raw_status - def compare_results(self): - self.clean() - expected = self.get_shell_results() - self.clean() - actual = self.get_extractor_results() + def compare_results(self, actual): posttest_result = self.get_posttest_result() self.clean() - if expected is None: - raise ExtractorTestError("could not get baseline results") - elif actual is None: - raise ExtractorTestError("could not get extractor results") - elif expected != actual: - result = self.show_status('FAILED') - print "Only in baseline results:" - print '\n'.join(expected.difference(actual)) - print "Only in actual results:" - print '\n'.join(actual.difference(expected)) + status, expected = self.get_shell_results() + self.clean() + if expected != actual: + print >>output_buffer, "Only in baseline results:" + print >>output_buffer, '\n'.join(expected.difference(actual)) + print >>output_buffer, "Only in actual results:" + print >>output_buffer, '\n'.join(actual.difference(expected)) + return self.show_status('FAILED') elif posttest_result != 0: - result = self.show_status('FAILED') - print "Posttest returned status code", posttest_result - else: - result = self.show_status('Passed') - return result + print >>output_buffer, "Posttest gave status code", posttest_result + return self.show_status('FAILED') + return self.show_status('Passed') + def have_error_mismatch(self, status): + if self.error and (status == 0): + return "x did not return expected error" + elif (not self.error) and (status != 0): + return "x returned error code %s" % (status,) + return None + + def grep_output(self): + output_buffer.seek(0, 0) + output_buffer.readline() + output = output_buffer.read(-1) + if self.grep and (not re.search(self.grep, output)): + return "output did not match %s" % (self.grep) + elif self.antigrep and re.search(self.antigrep, output): + return "output matched antigrep %s" % (self.antigrep) + return None + + def check_results(self): + output_buffer.seek(0, 0) + output_buffer.truncate() + self.clean() + status, actual = self.get_extractor_results() + problem = self.have_error_mismatch(status) or self.grep_output() + if problem: + return self.show_status('FAILED', problem) + return self.compare_results(actual) + def run(self): if self.directory: os.mkdir(self.directory) os.chdir(self.directory) try: - result = self.compare_results() + result = self.check_results() except ExtractorTestError, error: result = self.show_status('ERROR', error) if self.directory: os.chdir(ROOT_DIR) + subprocess.call(['chmod', '-R', '700', self.directory]) subprocess.call(['rm', '-rf', self.directory]) return result @@ -153,12 +184,14 @@ test_db.close() tests = [ExtractorTest(**data) for data in test_data] for original_data in test_data: - if original_data.has_key('directory'): + if (original_data.has_key('directory') or + (not original_data.has_key('baseline'))): continue data = original_data.copy() data['name'] += ' in ..' data['directory'] = 'inside-dir' - data['filename'] = os.path.join('..', data['filename']) + data['filenames'] = ' '.join(['../%s' % filename for filename in + data.get('filenames', '').split()]) tests.append(ExtractorTest(**data)) results = [test.run() for test in tests] counts = {} @@ -167,3 +200,4 @@ for result in results: counts[result] += 1 print " Totals:", ', '.join(["%s %s" % (counts[key], key) for key in OUTCOMES]) +output_buffer.close() diff -r 29794d4d41aa -r 481a2b4be471 tests/tests.yml --- a/tests/tests.yml Sun Dec 31 19:27:23 2006 -0500 +++ b/tests/tests.yml Sun Dec 31 19:29:46 2006 -0500 @@ -1,29 +1,29 @@ - name: basic .tar - filename: test-1.23.tar + filenames: test-1.23.tar baseline: | tar -xf $1 - name: basic .tar.gz - filename: test-1.23.tar.gz + filenames: test-1.23.tar.gz baseline: | tar -zxf $1 - name: basic .tar.bz2 - filename: test-1.23.tar.bz2 + filenames: test-1.23.tar.bz2 baseline: | mkdir test-1.23 cd test-1.23 tar -jxf ../$1 - name: basic .zip - filename: test-1.23.zip + filenames: test-1.23.zip baseline: | mkdir test-1.23 cd test-1.23 unzip -q ../$1 - name: basic .deb - filename: test-1.23_all.deb + filenames: test-1.23_all.deb baseline: | TD=$PWD mkdir test-1.23 @@ -34,7 +34,7 @@ rm /tmp/data.tar.gz - name: recursion and permissions - filename: test-recursive-badperms.tar.bz2 + filenames: test-recursive-badperms.tar.bz2 options: -r baseline: | mkdir test-recursive-badperms @@ -50,26 +50,26 @@ - name: decompression directory: inside-dir - filename: ../test-text.gz + filenames: ../test-text.gz baseline: | zcat $1 >test-text - name: decompression with -r directory: inside-dir - filename: ../test-text.gz + filenames: ../test-text.gz options: -r baseline: | zcat $1 >test-text - name: decompression with -fr directory: inside-dir - filename: ../test-text.gz + filenames: ../test-text.gz options: -fr baseline: | zcat $1 >test-text - name: overwrite protection - filename: test-1.23.tar.bz2 + filenames: test-1.23.tar.bz2 baseline: | mkdir test-1.23 test-1.23.1 cd test-1.23.1 @@ -78,7 +78,7 @@ mkdir test-1.23 - name: overwrite option - filename: test-1.23.tar.bz2 + filenames: test-1.23.tar.bz2 options: -o baseline: | mkdir test-1.23 @@ -89,14 +89,14 @@ - name: flat option directory: inside-dir - filename: ../test-1.23.tar.bz2 + filenames: ../test-1.23.tar.bz2 options: -f baseline: | tar -jxf $1 - name: flat recursion and permissions directory: inside-dir - filename: ../test-recursive-badperms.tar.bz2 + filenames: ../test-recursive-badperms.tar.bz2 options: -fr baseline: | tar -jxf $1 @@ -105,3 +105,48 @@ posttest: | if [ "x`cat testdir/testfile`" != "xhey" ]; then exit 1; fi +- name: no files + error: true + grep: "[Uu]sage" + +- name: bad file + error: true + filenames: nonexistent-file.tar + +- name: not an archive + error: true + filenames: tests.yml + +- name: bad options + options: --nonexistent-option + filenames: test-1.23.tar + error: true + +- name: --version + options: --version + grep: ersion \d+\.\d+ + filenames: test-1.23.tar + baseline: | + exit 0 + +- name: one good archive of many + filenames: tests.yml test-1.23.tar nonexistent-file.tar + error: true + baseline: | + tar -xf $2 + +- name: silence + filenames: tests.yml + options: -qq + error: true + antigrep: . + +- name: can't write to directory + directory: inside-dir + filenames: ../test-1.23.tar + error: true + grep: ERROR + antigrep: Traceback + prerun: | + chmod 500 . +