From 386e2658e83dc4bc27fa921499b4445bb8ac3d60 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 29 Jul 2017 18:36:49 +0300 Subject: Add: scenario for concurrent building of projects --- yarns/200-build.yarn | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++- yarns/lib.py | 23 +++++- 2 files changed, 218 insertions(+), 3 deletions(-) (limited to 'yarns') diff --git a/yarns/200-build.yarn b/yarns/200-build.yarn index 40d0d08..c4a9f73 100644 --- a/yarns/200-build.yarn +++ b/yarns/200-build.yarn @@ -1,11 +1,14 @@ Building a project ============================================================================= -This chapter uses the controller to walk through all the steps for a +One build, one worker +----------------------------------------------------------------------------- + +This section uses the controller to walk through all the steps for a build. We start with some setup, defining a git repo and an ick project and starting the controller. - SCENARIO run a build + SCENARIO run a build with one worker GIVEN a git repo foo.git with file index.mdwn containing ... "hello, world\n" @@ -109,3 +112,194 @@ previous log will contain the previously current log. And we're done. FINALLY stop controller instance + + +Two builds, two workers +----------------------------------------------------------------------------- + +This section runs two builds using two workers. The primary goal here +is to make sure each worker gets consecutive steps for its own build, +so that it runs all the steps in the same workspace. + +We start with some setup, defining a git repo and an ick project and +starting the controller. + + SCENARIO run two builds with two workers + + GIVEN a git repo foo.git with file index.mdwn containing + ... "hello, world\n" + AND a git repo bar.git with file index.mdwn containing + ... "hello, bar\n" + AND a project foo, using foo.git, publishing to foo-web + AND a project bar, using bar.git, publishing to bar-web + AND a running controller instance + +Ensure controller knows of both projects. The list of project names +should be sorted alphabetically. + + WHEN user calls GET /projects + THEN response has status 200, + ... and JSON body "{ "projects": [ "bar", "foo" ] }" + +There is no job running now, so if the worker manager asks for work, +it gets nothing. + + WHEN worker manager calls GET /worker/one + THEN response has status 200, and an empty body + +Trigger new builds on both projects. There is now work to do. + + WHEN git server calls GET /projects/foo/+trigger + THEN response has status 200 + + WHEN worker manager calls GET /worker/one + THEN response has status 200, and JSON body "{ + ... "project": "foo", + ... "git": "foo.git", + ... "shell": "ikiwiki --build" + ... }" + +The second worker should still get nothing to do. + + WHEN worker manager calls GET /worker/two + THEN response has status 200, and an empty body + +Trigger the other project, and the second worker gets something to do. + + WHEN git server calls GET /projects/bar/+trigger + THEN response has status 200 + + WHEN worker manager calls GET /worker/two + THEN response has status 200, and JSON body "{ + ... "project": "bar", + ... "git": "foo.git", + ... "shell": "ikiwiki --build" + ... }" + +Pretend the build step for the first project is running, and send +output to the controller. Don't send an exit code, since the pretend +step hasn't finished. Check that the pretend output we sent ends up in +the current build log. + + WHEN worker manager calls POST /worker/one/snippet, + ... with JSON body '{ + ... "project": "foo", + ... "stdout": "ikiwiki build output", + ... "stderr": "", + ... "exit-code": null + ... }' + AND user calls GET /projects/foo/logs/current + THEN response has status 200, and text body "ikiwiki build output" + +The current build step hasn't changed. + + WHEN worker manager calls GET /worker/one + THEN response has status 200, + ... and JSON body "{ + ... "project": "foo", + ... "git": "foo.git", + ... "shell": "ikiwiki --build" + ... }" + +Pretend the build step finishes. Make sure current log updates, and +that we get a new thing to run. + + WHEN worker manager calls POST /worker/one/snippet, + ... with JSON body '{ + ... "project": "foo", + ... "stdout": "|more output", + ... "stderr": "", + ... "exit-code": 0 + ... }' + AND user calls GET /projects/foo/logs/current + THEN response has status 200, and an empty body + + WHEN user calls GET /projects/foo/logs/previous + THEN response has status 200, + ... and text body "ikiwiki build output|more output" + + WHEN worker manager calls GET /worker/one + THEN response has status 200, + ... and JSON body "{ + ... "project": "foo", + ... "git": "foo.git", + ... "shell": "rsync" + ... }" + +The other worker is still running its step, and if it asks, it gets +the same build step to run. + + WHEN worker manager calls GET /worker/two + THEN response has status 200, and JSON body "{ + ... "project": "bar", + ... "git": "foo.git", + ... "shell": "ikiwiki --build" + ... }" + +Tell controller the rsync command of the first project also finishes. +After that, there should be nothing more to do. The current log should +become empty, the previous log will contain the previously current +log. + + WHEN worker manager calls POST /worker/one/snippet, + ... with JSON body '{ + ... "project": "foo", + ... "stdout": "rsync output", + ... "stderr": "", + ... "exit-code": 0 + ... }' + + WHEN user calls GET /projects/foo/logs/current + THEN response has status 200, and an empty body + + WHEN user calls GET /projects/foo/logs/previous + THEN response has status 200, and text body "rsync output" + + WHEN user calls GET /worker/one + THEN response has status 200, and an empty body + +Finish the other project build. + + WHEN worker manager calls POST /worker/two/snippet, + ... with JSON body '{ + ... "project": "bar", + ... "stdout": "ikiwiki output", + ... "stderr": "", + ... "exit-code": 0 + ... }' + AND user calls GET /projects/bar/logs/current + THEN response has status 200, and an empty body + + WHEN user calls GET /projects/bar/logs/previous + THEN response has status 200, + ... and text body "ikiwiki output" + + WHEN worker manager calls GET /worker/two + THEN response has status 200, + ... and JSON body "{ + ... "project": "foo", + ... "git": "foo.git", + ... "shell": "rsync" + ... }" + + WHEN worker manager calls POST /worker/two/snippet, + ... with JSON body '{ + ... "project": "bar", + ... "stdout": "second worker rsync output", + ... "stderr": "", + ... "exit-code": 0 + ... }' + + WHEN user calls GET /projects/bar/logs/current + THEN response has status 200, and an empty body + + WHEN user calls GET /projects/bar/logs/previous + THEN response has status 200, and text body + ... "second worker rsync output" + + WHEN user calls GET /worker/two + THEN response has status 200, and an empty body + +And we're done. + + FINALLY stop controller instance diff --git a/yarns/lib.py b/yarns/lib.py index 9040bb7..fc4acfc 100644 --- a/yarns/lib.py +++ b/yarns/lib.py @@ -68,7 +68,28 @@ def request(method, url, body=None): def parse_json(text): - return json.loads(text) + return json.loads(text, object_pairs_hook=dictify) + + + +def dictify(pairs): + return { + stringify(key): stringify(value) + for key, value in pairs + } + + +def stringify(x): + if isinstance(x, unicode): + return str(x) + if isinstance(x, list): + return [stringify(y) for y in x] + if isinstance(x, dict): + return { + stringify(key): stringify(value) + for key, value in pairs + } + return x def parse_yaml(text): -- cgit v1.2.1