main.lua

#!/usr/bin/env luajit

local json = require "cjson";
local time = require "socket".gettime;
local sleep = require "socket".sleep;

local is_scansion_error = require "scansion.error".is;
local new_error = require "scansion.error".new_error;

local result_log_filename = nil;
local server_log_reader = nil;
local test_metadata = {};
local server_log_wait_time = 0.2;
local skip_server_startup_log = false;
local action_timeout = 10;
local verse_log_levels = { "warn", "error" };
local quiet = false;
local force_summary = false;
local serve_mode = false;
local only_tags, skip_tags;

local property_rules = {};

local function apply_object_properties(class, name, object)
	for _, rule in ipairs(property_rules) do
		if (not(rule.class) or tostring(rule.class):lower() == tostring(class):lower())
		and (not(rule.name) or tostring(rule.name):lower() == tostring(name):lower()) then
			for prop_key, prop_value in pairs(rule.properties) do
				object[prop_key] = prop_value;
			end
		end
	end
end

local function process_options()
	local function get_value()
		return (assert(table.remove(arg, 1), "unexpected end of command-line options"));
	end

	local files = {};

	while arg[1] and arg[1]:sub(1,1) == "-" do
		local opt = arg[1];
		if opt == "--" then
			table.remove(arg, 1);
			break;
		end
		table.remove(arg, 1);
		if opt == "--port" or opt == "-p" then
			local port = assert(tonumber(get_value()), "port number must be a number");
			table.insert(property_rules, { class = "client", properties = { connect_port = port } });
		elseif opt == "--host" or opt == "-h" then
			local host = get_value();
			table.insert(property_rules, { class = "client", properties = { connect_host = host } });
		elseif opt == "--out" or opt == "-o" then
			result_log_filename = get_value();
		elseif opt == "--server-log" or opt == "-s" then
			local server_log = assert(io.open(get_value(), "r"));
			function server_log_reader()
				local new_lines = {};
				local last_line_time = time();
				while time() - last_line_time < server_log_wait_time do
					sleep(0.05);
					for line in server_log:lines() do
						table.insert(new_lines, line);
						last_line_time = time();
					end
				end
				return new_lines;
			end
		elseif opt == "--skip-server-startup-log" then
			skip_server_startup_log = true;
		elseif opt == "--step-timeout" then
			action_timeout = assert(tonumber(get_value()), "number expected for --step-timeout");
		elseif opt == "--meta" then
			local tag = get_value();
			local key, value = tag:match("^([^=]+):(.+)$");
			if key and value then
				test_metadata[key] = value;
			else
				error("Unable to parse metadata: "..tag);
			end
		elseif opt == "--with-tag" then
			if not only_tags then
				only_tags = {};
			end
			local tag = get_value();
			local key, value = tag:match("^([^=]+):(.+)$");
			table.insert(only_tags, { key or tag, value });
		elseif opt == "--skip-tag" then
			if not skip_tags then
				skip_tags = {};
			end
			local tag = get_value();
			local key, value = tag:match("^([^=]+):(.+)$");
			table.insert(skip_tags, { key or tag, value });
		elseif opt == "--verbose" or opt == "-v" then
			verse_log_levels = { "debug", "info", "warn", "error" };
			quiet = false;
		elseif opt == "--quiet" or opt == "-q" then
			verse_log_levels = { "error" };
			quiet = true;
		elseif opt == "--script-dir" or opt == "-d" then
			local have_lfs, lfs = pcall(require, "lfs");
			if not have_lfs then
				error("The '-d' parameter requires LuaFileSystem (lfs), please make sure this library is available");
			end
			local path = assert(get_value(), "path expected for '-d'"):gsub("/*$", "");
			for f in lfs.dir(path) do
				if f:sub(1,1) ~= "." and f:sub(-4) == ".scs" then
					table.insert(files, path.."/"..f);
				end
			end
		elseif opt == "--summary" then
			force_summary = true;
		elseif opt == "--serve" then
			serve_mode = 8007;
		elseif opt == "--serve-port" then
			serve_mode = assert(tonumber(get_value()), "expected port number");
		else
			error("Unhandled command-line option: "..opt);
		end
	end
	for _, file in ipairs(arg) do
		table.insert(files, file);
	end
	return files;
end

local function read_script(script_name)
	io.input(script_name);
	return io.read("*a");
end

local function parse_script(data)
	local parser = require "scansion.parser";
	return assert(parser.parse(data));
end

local function initialize_script(script)
	local c = 0;
	for name, object in pairs(script.objects) do --luacheck: ignore name
		local o = require("scansion.objects."..object.type);
		object.handler = o;
		object.script = script;
		apply_object_properties(object.type, object.name, object);
		if object.type == "client" and not object.stanza_timeout then
			object.stanza_timeout = action_timeout - 1;
		end
		o._validate(object);
		c = c + 1;
	end
end

local function uninitialize_script(script)
	script.finished = true;
	for name, obj in pairs(script.objects) do --luacheck: ignore name
		if obj.handler._finish then
			obj.handler._finish(obj);
		end
	end
end

local function initialize_verse(errcb)
	local verse = require "verse";

	verse.set_log_handler(verse._default_log_handler, verse_log_levels);

	local function error_handler(err)
		-- Only log actual errors. Those reported by Scansion will be handled gracefully higher up.
		if not is_scansion_error(err) then
			verse.log("error", "Error: %s", err);
			verse.log("error", "Traceback: %s", debug.traceback());
		end
		errcb(err);
	end
	verse.set_error_handler(error_handler);
	return verse;
end

local function main(log_data, script)

	local ok, err = true;

	initialize_script(script);

	if server_log_reader then
		if skip_server_startup_log then
			server_log_reader();
		else
			log_data("server", { lines = server_log_reader() });
		end
	end

	local verse = initialize_verse(function (_err) ok, err = false, _err end);
	local async = require "scansion.async";

	local runner = async.runner(function ()
		for i, action in ipairs(script.actions) do
			local object = script.objects[action.object_name];
			local handler = object.handler;
			assert(handler[action.action], "Objects of type '"..object.type.."' do not support action '"..action.action.."'");

			if server_log_reader then
				log_data("server", { lines = server_log_reader() });
			end
			log_data("action", {
				action = action.action;
				object = object.name;
				object_type = object.type;
				extra = action.extra;
				annotation = action.annotation;
			});

			if action_timeout and action_timeout > 0 then
				local action_number = i;
				verse.add_task(action_timeout, function ()
					if i == action_number and not script.finished then
						local e = new_error("action-timeout", { text = "Timeout waiting for response" });
						error(e);
					end
				end);
			end
			do
				--luacheck: ignore ok err
				local ok, err = pcall(handler[action.action], object, action.extra);
				if server_log_reader then
					log_data("server", { lines = server_log_reader() });
				end
				if not ok then
					log_data("error", { message = err });
					error(err);
				end
			end
		end
		script.finished = true;
		verse.log("info", "Completed script!");
		verse.quit();
	end, {
		error = function (runner, _err) --luacheck: ignore runner
			verse.log("error", "Runner caught error: %s", _err);
			ok, err = false, _err;
			verse.quit();
		end;
	});

	runner:run(true);
	verse.log("debug", "runner paused")

	verse.loop();

	if server_log_reader then
		log_data("server", { lines = server_log_reader() });
	end

	uninitialize_script(script);

	return ok, err;
end

-- Process command-line options
local files = process_options();

local console_handlers = require "scansion.console".new({
	summary = not(quiet) or force_summary;
	quiet = quiet;
});

local function console_logger(event, data)
	local h = console_handlers[event];
	if h then
		io.write(h(data), "\n");
	end
end


local result_log;
if result_log_filename then
	result_log = assert(io.open(result_log_filename, "w+"));
	result_log:write("[\n");
end

local function log_data(type, data)
	console_logger(type, data);

	if result_log then
		local entry = { type = type, data = data, time = time() };
		result_log:write("  ", json.encode(entry), type ~= "end" and ",\n" or "\n");
		result_log:flush();
		if type == "end" then
			result_log:write("]\n");
			result_log:close();
			result_log = nil;
		end
	end
end;

local function run_test_script(script_name, script_text)
	local script = parse_script(script_text)
	local script_data = { title = script.title, summary = script.summary, tags = script.tags, filename = script_name };

	local tags = script.tags or {};
	if skip_tags then
		for _, skip in ipairs(skip_tags) do
			local k, v = skip[1], skip[2];
			if tags[k] and (not(v) or tags[k] == v) then
				return { name = script.title or script.name, status = "skipped", reason = "skipping tag "..k };
			end
		end
	end
	if only_tags then
		for _, only in ipairs(only_tags) do
			local k, v = only[1], only[2];
			if not tags[k] or (v and tags[k] ~= v) then
				print(k, v, tags[k])
				return { name = script.title or script.name, status = "skipped", reason = "not tagged "..k };
			end
		end
	end

	log_data("script", script_data);

	local ok, result, err = pcall(main, log_data, script);

	local event, status, reason;
	if not ok then
		event, status, reason = "test-error", "error", result;
	elseif not result then
		event, status, reason = "test-failed", "fail", err;
	else
		event, status, reason = "test-passed", "ok", nil;
	end

	local result_data = { name = script.title or script_name, status = status, reason = reason };

	log_data(event, result_data);

	return result_data;
end


if serve_mode then
	local have_serve, serve = pcall(require, "scansion.serve");

	if not have_serve then
		error("This version of scansion was not built with --with-server");
	end

	initialize_verse(function (e) --luacheck: ignore e
		-- This function handles scansion errors,
		-- but they shouldn't reach here anyway
	end);
	serve.run({ port = serve_mode }, run_test_script);
	os.exit(0);
end

log_data("start", { metadata = test_metadata });

local result_tally = { all = {} };

assert(#files > 0, "No test script provided");
for i = 1, #files do
	local script_text = read_script(files[i]);
	local ret = run_test_script(files[i], script_text);
	if not result_tally[ret.status] then
		result_tally[ret.status] = {};
	end
	table.insert(result_tally[ret.status], ret);
	table.insert(result_tally.all, ret);
end

log_data("end", { time = time(), summary = result_tally });


local exit_code = 0;
if result_tally.error then
	exit_code = 2;
elseif result_tally.fail then
	exit_code = 1;
end

os.exit(exit_code);