Tue, 11 Sep 2018 23:25:51 +0100
main: Modify run_test_script() to take the actual script data
This allows scripts to come from places other than files.
#!/usr/bin/env luajit local json = require "cjson"; local time = require "socket".gettime; local sleep = require "socket".sleep; local is_scansion_error = require "scansion.error".is; local new_error = require "scansion.error".new_error; local result_log_filename = nil; local server_log_reader = nil; local test_metadata = {}; local server_log_wait_time = 0.2; local skip_server_startup_log = false; local action_timeout = 5; local verse_log_levels = { "warn", "error" }; local quiet = false; local force_summary = false; local property_rules = {}; local function apply_object_properties(class, name, object) for _, rule in ipairs(property_rules) do if (not(rule.class) or tostring(rule.class):lower() == tostring(class):lower()) and (not(rule.name) or tostring(rule.name):lower() == tostring(name):lower()) then for prop_key, prop_value in pairs(rule.properties) do object[prop_key] = prop_value; end end end end local function process_options() local function get_value() return (assert(table.remove(arg, 1), "unexpected end of command-line options")); end local files = {}; while arg[1] and arg[1]:sub(1,1) == "-" do local opt = arg[1]; if opt == "--" then table.remove(arg, 1); break; end table.remove(arg, 1); if opt == "--port" or opt == "-p" then local port = assert(tonumber(get_value()), "port number must be a number"); table.insert(property_rules, { class = "client", properties = { connect_port = port } }); elseif opt == "--host" or opt == "-h" then local host = get_value(); table.insert(property_rules, { class = "client", properties = { connect_host = host } }); elseif opt == "--out" or opt == "-o" then result_log_filename = get_value(); elseif opt == "--server-log" or opt == "-s" then local server_log = assert(io.open(get_value(), "r")); function server_log_reader() local new_lines = {}; local last_line_time = time(); while time() - last_line_time < server_log_wait_time do sleep(0.05); for line in server_log:lines() do table.insert(new_lines, line); last_line_time = time(); end end return new_lines; end elseif opt == "--skip-server-startup-log" then skip_server_startup_log = true; elseif opt == "--step-timeout" then action_timeout = assert(tonumber(get_value()), "number expected for --step-timeout"); elseif opt == "--tag" then local tag = get_value(); local key, value = tag:match("^([^=]+):(.+)$"); if key and value then test_metadata[key] = value; else error("Unable to parse tag: "..tag); end elseif opt == "--verbose" or opt == "-v" then verse_log_levels = { "debug", "info", "warn", "error" }; quiet = false; elseif opt == "--quiet" or opt == "-q" then verse_log_levels = { "error" }; quiet = true; elseif opt == "--script-dir" or opt == "-d" then local have_lfs, lfs = pcall(require, "lfs"); if not have_lfs then error("The '-d' parameter requires LuaFileSystem (lfs), please make sure this library is available"); end local path = assert(get_value(), "path expected for '-d'"):gsub("/*$", ""); for f in lfs.dir(path) do if f:sub(1,1) ~= "." then table.insert(files, path.."/"..f); end end elseif opt == "--summary" then force_summary = true; else error("Unhandled command-line option: "..opt); end end for _, file in ipairs(arg) do table.insert(files, file); end return files; end local function read_script(script_name) io.input(script_name); return io.read("*a"); end local function parse_script(data) local parser = require "scansion.parser"; return assert(parser.parse(data)); end local function initialize_script(script) local c = 0; for name, object in pairs(script.objects) do --luacheck: ignore name local o = require("scansion.objects."..object.type); object.handler = o; object.script = script; apply_object_properties(object.type, object.name, object); if object.type == "client" and not object.stanza_timeout then object.stanza_timeout = action_timeout - 1; end o._validate(object); c = c + 1; end end local function uninitialize_script(script) script.finished = true; for name, obj in pairs(script.objects) do if obj.handler._finish then obj.handler._finish(obj); end end end local function initialize_verse(errcb) local verse = require "verse"; verse.set_log_handler(verse._default_log_handler, verse_log_levels); local function error_handler(err) -- Only log actual errors. Those reported by Scansion will be handled gracefully higher up. if not is_scansion_error(err) then verse.log("error", "Error: %s", err); verse.log("error", "Traceback: %s", debug.traceback()); end errcb(err); end verse.set_error_handler(error_handler); return verse; end local function main(log_data, script) local ok, err = true; initialize_script(script); if server_log_reader then if skip_server_startup_log then server_log_reader(); else log_data("server", { lines = server_log_reader() }); end end local verse = initialize_verse(function (_err) ok, err = false, _err end); local async = require "scansion.async"; local runner = async.runner(function () for i, action in ipairs(script.actions) do local object = script.objects[action.object_name]; local handler = object.handler; assert(handler[action.action], "Objects of type '"..object.type.."' do not support action '"..action.action.."'"); if server_log_reader then log_data("server", { lines = server_log_reader() }); end log_data("action", { action = action.action; object = object.name; object_type = object.type; extra = action.extra; annotation = action.annotation; }); if action_timeout and action_timeout > 0 then local action_number = i; verse.add_task(action_timeout, function () if i == action_number and not script.finished then local e = new_error("action-timeout", { text = "Timeout waiting for response" }); error(e); end end); end do --luacheck: ignore ok err local ok, err = pcall(handler[action.action], object, action.extra); if server_log_reader then log_data("server", { lines = server_log_reader() }); end if not ok then log_data("error", { message = err }); error(err); end end end script.finished = true; verse.log("info", "Completed script!"); verse.quit(); end, { error = function (runner, _err) --luacheck: ignore runner verse.log("error", "Runner caught error: %s", _err); ok, err = false, _err; verse.quit(); end; }); runner:run(true); verse.log("debug", "runner paused") verse.loop(); if server_log_reader then log_data("server", { lines = server_log_reader() }); end uninitialize_script(script); return ok, err; end -- Process command-line options local files = process_options(); local console_handlers = require "scansion.console".new({ summary = not(quiet) or force_summary; quiet = quiet; }); local function console_logger(event, data) local h = console_handlers[event]; if h then io.write(h(data), "\n"); end end local result_log; if result_log_filename then result_log = assert(io.open(result_log_filename, "w+")); result_log:write("[\n"); end local function log_data(type, data) console_logger(type, data); if result_log then local entry = { type = type, data = data, time = time() }; result_log:write(" ", json.encode(entry), type ~= "end" and ",\n" or "\n"); result_log:flush(); if type == "end" then result_log:write("]\n"); result_log:close(); result_log = nil; end end end; local function run_test_script(script_name, script_text) local script = parse_script(script_text) log_data("script", { title = script.title, summary = script.summary, tags = script.tags, filename = script_name }); local ok, result, err = pcall(main, log_data, script); local status, reason = "ok"; if not ok then status, reason = "error", result; log_data("test-error", { error = result }); elseif not result then status, reason = "fail", err; log_data("test-failed", { error = err }); else log_data("test-passed"); end return { name = script.title or script_name, status = status, reason = reason }; end log_data("start", { metadata = test_metadata }); local result_tally = { all = {} }; assert(#files > 0, "No test script provided"); for i = 1, #files do local script_text = read_script(files[i]); local ret = run_test_script(files[i], script_text); if not result_tally[ret.status] then result_tally[ret.status] = {}; end table.insert(result_tally[ret.status], ret); table.insert(result_tally.all, ret); end log_data("end", { time = time(), summary = result_tally }); local exit_code = 0; if result_tally.error then exit_code = 2; elseif result_tally.fail then exit_code = 1; end os.exit(exit_code);