main.lua

Mon, 10 Sep 2018 10:02:42 +0100

author
Matthew Wild <mwild1@gmail.com>
date
Mon, 10 Sep 2018 10:02:42 +0100
changeset 108
cd58c64c67c0
parent 107
65791ba388b9
child 109
f7609d1a5bdf
permissions
-rwxr-xr-x

scansion: Add support for passing multiple scripts on the command-line

54
09a754d5fcc1 main: Change shebang to LuaJIT
Kim Alvefur <zash@zash.se>
parents: 46
diff changeset
1 #!/usr/bin/env luajit
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
2
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
3 local json = require "cjson";
61
21871fb2db99 main: Switch to socket.gettime for higher accuracy timestamps in machine-readable logs
Matthew Wild <mwild1@gmail.com>
parents: 59
diff changeset
4 local time = require "socket".gettime;
67
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
5 local sleep = require "socket".sleep;
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
6
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
7 local result_log_filename = nil;
62
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
8 local server_log_reader = nil;
69
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
9 local test_metadata = {};
67
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
10 local server_log_wait_time = 0.2;
76
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
11 local skip_server_startup_log = false;
82
f90056b8e278 main.lua, client: Make timeouts more coherent (stanza timeout was greater than action timeout), add command-line options to change them
Matthew Wild <mwild1@gmail.com>
parents: 81
diff changeset
12 local action_timeout = 5;
107
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
13 local verse_log_levels = { "warn", "error" };
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
14 local quiet = false;
67
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
15
56
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
16 local property_rules = {};
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
17
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
18 local function apply_object_properties(class, name, object)
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
19 for _, rule in ipairs(property_rules) do
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
20 if (not(rule.class) or tostring(rule.class):lower() == tostring(class):lower())
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
21 and (not(rule.name) or tostring(rule.name):lower() == tostring(name):lower()) then
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
22 for prop_key, prop_value in pairs(rule.properties) do
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
23 object[prop_key] = prop_value;
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
24 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
25 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
26 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
27 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
28
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
29 function process_options()
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
30 local function get_value()
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
31 return (assert(table.remove(arg, 1), "unexpected end of command-line options"));
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
32 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
33 while arg[1] and arg[1]:sub(1,1) == "-" do
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
34 local opt = arg[1];
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
35 if opt == "--" then
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
36 table.remove(arg, 1);
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
37 break;
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
38 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
39 table.remove(arg, 1);
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
40 if opt == "--port" or opt == "-p" then
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
41 local port = assert(tonumber(get_value()), "port number must be a number");
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
42 table.insert(property_rules, { class = "client", properties = { connect_port = port } });
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
43 elseif opt == "--host" or opt == "-h" then
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
44 local host = get_value();
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
45 table.insert(property_rules, { class = "client", properties = { connect_host = host } });
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
46 elseif opt == "--out" or opt == "-o" then
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
47 result_log_filename = get_value();
62
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
48 elseif opt == "--server-log" or opt == "-s" then
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
49 local server_log = assert(io.open(get_value(), "r"));
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
50 function server_log_reader()
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
51 local new_lines = {};
67
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
52 local last_line_time = time();
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
53 while time() - last_line_time < server_log_wait_time do
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
54 sleep(0.05);
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
55 for line in server_log:lines() do
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
56 table.insert(new_lines, line);
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
57 last_line_time = time();
a2a9dd606200 main: Add delay when reading server logs
Matthew Wild <mwild1@gmail.com>
parents: 65
diff changeset
58 end
62
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
59 end
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
60 return new_lines;
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
61 end
76
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
62 elseif opt == "--skip-server-startup-log" then
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
63 skip_server_startup_log = true;
82
f90056b8e278 main.lua, client: Make timeouts more coherent (stanza timeout was greater than action timeout), add command-line options to change them
Matthew Wild <mwild1@gmail.com>
parents: 81
diff changeset
64 elseif opt == "--step-timeout" then
f90056b8e278 main.lua, client: Make timeouts more coherent (stanza timeout was greater than action timeout), add command-line options to change them
Matthew Wild <mwild1@gmail.com>
parents: 81
diff changeset
65 action_timeout = assert(tonumber(get_value()), "number expected for --step-timeout");
69
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
66 elseif opt == "--tag" then
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
67 local tag = get_value();
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
68 local key, value = tag:match("^([^=]+):(.+)$");
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
69 if key and value then
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
70 test_metadata[key] = value;
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
71 else
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
72 error("Unable to parse tag: "..tag);
1de0ebd8832f main.lua: Support tagging metadata for test runs, and include it in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 68
diff changeset
73 end
107
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
74 elseif opt == "--verbose" or opt == "-v" then
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
75 verse_log_levels = { "debug", "info", "warn", "error" };
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
76 quiet = false;
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
77 elseif opt == "--quiet" or opt == "-q" then
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
78 verse_log_levels = { "error" };
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
79 quiet = true;
58
8fa0a464c727 main.lua: Error on unknown command-line options
Matthew Wild <mwild1@gmail.com>
parents: 57
diff changeset
80 else
8fa0a464c727 main.lua: Error on unknown command-line options
Matthew Wild <mwild1@gmail.com>
parents: 57
diff changeset
81 error("Unhandled command-line option: "..opt);
56
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
82 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
83 end
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
84 assert(#arg > 0, "No test script provided");
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
85 end
0
2e31b584f8d9 It is better to write and run incomplete tests than not to run complete tests. -- Martin Fowler
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
86
108
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
87 function read_script(script_name)
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
88 io.input(script_name);
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
89 return io.read("*a");
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
90 end
0
2e31b584f8d9 It is better to write and run incomplete tests than not to run complete tests. -- Martin Fowler
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
91
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
92 function parse_script(data)
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
93 local parser = require "scansion.parser";
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
94 return assert(parser.parse(data));
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
95 end
0
2e31b584f8d9 It is better to write and run incomplete tests than not to run complete tests. -- Martin Fowler
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
96
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
97 function initialize_script(script)
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
98 local c = 0;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
99 for name, object in pairs(script.objects) do
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
100 local o = require("scansion.objects."..object.type);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
101 object.handler = o;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
102 object.script = script;
56
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
103 apply_object_properties(object.type, object.name, object);
82
f90056b8e278 main.lua, client: Make timeouts more coherent (stanza timeout was greater than action timeout), add command-line options to change them
Matthew Wild <mwild1@gmail.com>
parents: 81
diff changeset
104 if object.type == "client" and not object.stanza_timeout then
f90056b8e278 main.lua, client: Make timeouts more coherent (stanza timeout was greater than action timeout), add command-line options to change them
Matthew Wild <mwild1@gmail.com>
parents: 81
diff changeset
105 object.stanza_timeout = action_timeout - 1;
f90056b8e278 main.lua, client: Make timeouts more coherent (stanza timeout was greater than action timeout), add command-line options to change them
Matthew Wild <mwild1@gmail.com>
parents: 81
diff changeset
106 end
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
107 o._validate(object);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
108 c = c + 1;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
109 end
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
110
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
111 --print("Script defines "..c.." objects, and "..#script.actions.." actions");
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
112 return script;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
113 end
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
114
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
115 function initialize_verse(errcb)
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
116 local verse = require "verse";
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
117
107
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
118 verse.set_log_handler(verse._default_log_handler, verse_log_levels);
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
119
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
120 local function error_handler(err)
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
121 verse.log("error", "Error: %s", err);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
122 verse.log("error", "Traceback: %s", debug.traceback());
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
123 errcb(err);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
124 end
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
125 verse.set_error_handler(error_handler);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
126 return verse;
0
2e31b584f8d9 It is better to write and run incomplete tests than not to run complete tests. -- Martin Fowler
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
127 end
2e31b584f8d9 It is better to write and run incomplete tests than not to run complete tests. -- Martin Fowler
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
128
108
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
129 function main(log_data, script_name)
56
5eda634ea15b main.lua: Command-line processing, and a way to set connect_host/connect_port from the command-line
Matthew Wild <mwild1@gmail.com>
parents: 55
diff changeset
130
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
131 local ok, err = true;
0
2e31b584f8d9 It is better to write and run incomplete tests than not to run complete tests. -- Martin Fowler
Matthew Wild <mwild1@gmail.com>
parents:
diff changeset
132
108
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
133 local script = initialize_script(parse_script(read_script(script_name)));
71
f7dbb46b5770 main.lua: Log script title and summary
Matthew Wild <mwild1@gmail.com>
parents: 69
diff changeset
134
78
54bb54fe9ed2 main, scansion.parser: Allow scripts to include tags in comments at the beginning, which are included in JSON output
Matthew Wild <mwild1@gmail.com>
parents: 76
diff changeset
135 log_data("script", { title = script.title, summary = script.summary, tags = script.tags });
71
f7dbb46b5770 main.lua: Log script title and summary
Matthew Wild <mwild1@gmail.com>
parents: 69
diff changeset
136
62
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
137 if server_log_reader then
76
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
138 if skip_server_startup_log then
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
139 server_log_reader();
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
140 else
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
141 log_data("server", { lines = server_log_reader() });
fecc1af937be main.lua: Add --skip-server-startup-log to discard server log output at startup
Matthew Wild <mwild1@gmail.com>
parents: 71
diff changeset
142 end
62
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
143 end
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
144
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
145 local verse = initialize_verse(function (_err) ok, err = false, _err end);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
146 local async = require "scansion.async";
7
ecac723bb6e1 main: Run actions in async runner
Matthew Wild <mwild1@gmail.com>
parents: 3
diff changeset
147
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
148 local runner = async.runner(function (d)
80
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
149 for i, action in ipairs(script.actions) do
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
150 local object = script.objects[action.object_name];
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
151 local handler = object.handler;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
152 assert(handler[action.action], "Objects of type '"..object.type.."' do not support action '"..action.action.."'");
107
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
153 if not quiet then
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
154 print("");
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
155 end
68
cd63f764d1bf main.lua: Restructure server log reading
Matthew Wild <mwild1@gmail.com>
parents: 67
diff changeset
156 if server_log_reader then
cd63f764d1bf main.lua: Restructure server log reading
Matthew Wild <mwild1@gmail.com>
parents: 67
diff changeset
157 log_data("server", { lines = server_log_reader() });
cd63f764d1bf main.lua: Restructure server log reading
Matthew Wild <mwild1@gmail.com>
parents: 67
diff changeset
158 end
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
159 log_data("action", {
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
160 action = action.action;
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
161 object = object.name;
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
162 object_type = object.type;
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
163 extra = action.extra;
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
164 annotation = action.annotation;
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
165 });
107
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
166 if not quiet then
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
167 if action.annotation then
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
168 print(action.annotation);
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
169 end
65791ba388b9 scansion: Add -q and -v for controlling log output
Matthew Wild <mwild1@gmail.com>
parents: 89
diff changeset
170 print(object.name.." "..action.action);
43
b37504fa3031 Add annotations to actions (by grabbing the preceding comment)
Matthew Wild <mwild1@gmail.com>
parents: 30
diff changeset
171 end
80
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
172 if action_timeout and action_timeout > 0 then
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
173 local action_number = i;
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
174 verse.add_task(action_timeout, function ()
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
175 if i == action_number then
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
176 error("Timeout waiting for response from server");
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
177 end
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
178 end);
6676218cc481 main.lua: Add per-action timeout
Matthew Wild <mwild1@gmail.com>
parents: 78
diff changeset
179 end
68
cd63f764d1bf main.lua: Restructure server log reading
Matthew Wild <mwild1@gmail.com>
parents: 67
diff changeset
180 local ok, err = pcall(handler[action.action], object, action.extra);
62
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
181 if server_log_reader then
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
182 log_data("server", { lines = server_log_reader() });
52a24deb0cc3 main: Support for --server-log/-s option, which reads the server's log file during a test and includes it in the JSON log
Matthew Wild <mwild1@gmail.com>
parents: 61
diff changeset
183 end
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
184 if not ok then
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
185 log_data("error", { message = err });
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
186 error(err);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
187 end
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
188 end
89
f353cd50c075 main.lua: Add flag for when script has finished
Matthew Wild <mwild1@gmail.com>
parents: 82
diff changeset
189 script.finished = true;
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
190 verse.log("info", "Completed script!");
30
3c2489e79074 main: Remove 5s delay on quitting (was a hack for debug purposes)
Matthew Wild <mwild1@gmail.com>
parents: 21
diff changeset
191 verse.quit();
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
192 end, {
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
193 error = function (runner, _err)
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
194 verse.log("error", "Runner caught error: %s", _err);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
195 ok, err = false, _err;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
196 verse.quit();
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
197 end;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
198 });
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
199
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
200 runner:run(true);
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
201 verse.log("debug", "runner paused")
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
202
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
203 verse.loop();
81
0ee626030acb main.lua: Read any remaining server logs after script completes (error or otherwise)
Matthew Wild <mwild1@gmail.com>
parents: 80
diff changeset
204
0ee626030acb main.lua: Read any remaining server logs after script completes (error or otherwise)
Matthew Wild <mwild1@gmail.com>
parents: 80
diff changeset
205 if server_log_reader then
0ee626030acb main.lua: Read any remaining server logs after script completes (error or otherwise)
Matthew Wild <mwild1@gmail.com>
parents: 80
diff changeset
206 log_data("server", { lines = server_log_reader() });
0ee626030acb main.lua: Read any remaining server logs after script completes (error or otherwise)
Matthew Wild <mwild1@gmail.com>
parents: 80
diff changeset
207 end
0ee626030acb main.lua: Read any remaining server logs after script completes (error or otherwise)
Matthew Wild <mwild1@gmail.com>
parents: 80
diff changeset
208
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
209 return ok, err;
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
210 end
7
ecac723bb6e1 main: Run actions in async runner
Matthew Wild <mwild1@gmail.com>
parents: 3
diff changeset
211
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
212 -- Process command-line options
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
213 process_options();
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
214
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
215 -- Dummy logging function, used if no log file set
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
216 local log_data = function () end;
3
3cc860a893d2 main: verse.loop()
Matthew Wild <mwild1@gmail.com>
parents: 0
diff changeset
217
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
218 local result_log;
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
219 if result_log_filename then
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
220 result_log = assert(io.open(result_log_filename, "w+"));
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
221 result_log:write("[\n");
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
222 function log_data(type, data)
61
21871fb2db99 main: Switch to socket.gettime for higher accuracy timestamps in machine-readable logs
Matthew Wild <mwild1@gmail.com>
parents: 59
diff changeset
223 local entry = { type = type, data = data, time = time() };
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
224 result_log:write(" ", json.encode(entry), ",\n");
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
225 result_log:flush();
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
226 end
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
227 end
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
228
108
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
229 function run_test_script(script_name)
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
230
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
231 log_data("start", { metadata = test_metadata });
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
232 local ok, result, err = pcall(main, log_data, script_name);
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
233
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
234 local exit_code = 0;
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
235 if not ok then
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
236 print("TEST ERROR:", result);
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
237 exit_code = 2;
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
238 log_data("test-error", { error = result });
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
239 elseif not result then
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
240 print("TEST FAILED", err);
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
241 exit_code = 1;
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
242 log_data("test-failed", { error = err });
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
243 elseif not quiet then
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
244 print("TEST PASSED");
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
245 log_data("test-passed");
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
246 end
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
247 return exit_code;
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
248 end
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
249
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
250 local exit_code = 0;
108
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
251 for i = 1, #arg do
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
252 local ret = run_test_script(arg[i]);
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
253 if ret > exit_code then
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
254 exit_code = ret;
cd58c64c67c0 scansion: Add support for passing multiple scripts on the command-line
Matthew Wild <mwild1@gmail.com>
parents: 107
diff changeset
255 end
21
8ac80da35408 main: Refactor into functions
Matthew Wild <mwild1@gmail.com>
parents: 14
diff changeset
256 end
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
257
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
258 if result_log then
63
0f19ffbe1519 main: Separate logging for the last event to ensure proper JSON syntax
Matthew Wild <mwild1@gmail.com>
parents: 62
diff changeset
259 result_log:write([[ {"type": "end", "time": ]]..time().."}\n]\n");
57
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
260 result_log:close();
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
261 end
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
262
6407a0157518 main.lua: Add support for JSON log output to a file, for a machine-readable transcript of the test results
Matthew Wild <mwild1@gmail.com>
parents: 56
diff changeset
263 os.exit(exit_code);

mercurial