diff options
| -rwxr-xr-x | controller | 55 | ||||
| -rw-r--r-- | yarns/000.yarn | 24 | ||||
| -rw-r--r-- | yarns/100-hello.yarn | 9 | ||||
| -rw-r--r-- | yarns/200-build.yarn | 105 | ||||
| -rw-r--r-- | yarns/900.yarn | 94 | ||||
| -rw-r--r-- | yarns/lib.py | 63 |
6 files changed, 321 insertions, 29 deletions
@@ -7,15 +7,69 @@ import sys import bottle +current_log = '' +previous_log = '' +triggered = 0 + @bottle.route('/') def root(): + log('GET /') return 'This is the root' @bottle.route('/version') def version(): + log('GET /version') return { 'version': '1.0' } +@bottle.route('/projects') +def projects(): + return {'projects': ['foo']} + +@bottle.route('/worker/bar') +def worker(): + log('GET /worker/bar') + if triggered == 1: + return { + 'project': 'foo', + 'shell': 'ikiwiki --build', + } + elif triggered == 2: + return { + 'project': 'foo', + 'shell': 'rsync', + } + +@bottle.route('/projects/foo/logs/current') +def current(): + log('GET current log') + return current_log + +@bottle.route('/projects/foo/logs/previous') +def previous(): + log('GET previous log') + return previous_log + +@bottle.route('/projects/foo/+trigger') +def current(): + log('GET +trigger') + global triggered + triggered = True + return + +@bottle.post('/worker/bar/snippet') +def snippet(): + global current_log, previous_log, triggered + log('POST snippet') + obj = bottle.request.json + log('body: {}'.format(obj)) + current_log += obj['stdout'] + obj['stderr'] + if obj['exit-code'] is not None: + log('exit-code is {}, not None, rotating logs'.format(obj['exit-code'])) + previous_log = current_log + current_log = '' + triggered += 1 + # Command line args. @@ -42,4 +96,5 @@ with open(port_file, 'w') as f: f.write('{}\n'.format(port)) +log('starting daemon on port {}'.format(port)) bottle.run(port=port, quiet=True) diff --git a/yarns/000.yarn b/yarns/000.yarn new file mode 100644 index 0000000..7a19824 --- /dev/null +++ b/yarns/000.yarn @@ -0,0 +1,24 @@ +--- +title: Ick2 controller yarns +... + + +Introduction +============================================================================= + +Ick2 will be a continuous integration system. Its core component is +the **controller**, which does nothing, except decides what workers should +do. It knows of the projects that can be built, and keeps track what +step is being run on each project, and collects build output from the +workers. + +This document specifies the yarn test scenarios for a minimal viable +version of the controller: there will be a small number of project, +each project builds a web site from source in git, using ikiwiki, and +publishes the website on a server using rsync. + +The controller provides an HTTP API for controlling a build. The API +is used by an external entity (such as the git server) to trigger a +build, and by workers or worker proxies to request something to do, +and to report results. The API may also eventually be used by end +users to query status and results. diff --git a/yarns/100-hello.yarn b/yarns/100-hello.yarn index 6eeecfd..d719497 100644 --- a/yarns/100-hello.yarn +++ b/yarns/100-hello.yarn @@ -6,11 +6,10 @@ stop the controller, and make requests to it. SCENARIO controller smoke test GIVEN a running controller instance - WHEN client makes request GET /version - THEN HTTP status code is 200 - AND result matches { "version": "1.0" } + WHEN user calls GET /version + THEN response has status 200, and JSON body "{ "version": "1.0" }" - WHEN client makes request GET /blatherskite - THEN HTTP status code is 404 + WHEN user calls GET /blatherskite + THEN response has status 404 FINALLY stop controller instance diff --git a/yarns/200-build.yarn b/yarns/200-build.yarn new file mode 100644 index 0000000..77daebc --- /dev/null +++ b/yarns/200-build.yarn @@ -0,0 +1,105 @@ +Building a project +============================================================================= + +This chapter uses the controller to walk through all the steps for a +build. We start with some setup, defining a git repo and an ick +project and starting the controller. + + SCENARIO run a build + + GIVEN a git repo foo.git with file index.mdwn containing + ... "hello, world\n" + AND a project foo, using foo.git, publishing to foo-web + AND a running controller instance + +Ensure controller knows of the project. + + WHEN user calls GET /projects + THEN response has status 200, + ... and JSON body "{ "projects": [ "foo" ] }" + +There is no job running now, so if the worker manager asks for work, +it gets nothing. + + WHEN worker manager calls GET /worker/bar + THEN response has status 200, and an empty body + +Trigger a new build. There is now work to do. + + WHEN git server calls GET /projects/foo/+trigger + THEN response has status 200 + + WHEN worker manager calls GET /worker/bar + THEN response has status 200, and JSON body "{ + ... "project": "foo", + ... "shell": "ikiwiki --build" + ... }" + +Pretend a job is running, and send output to the controller. Don't send +an exit code, since the pretend job hasn't finished. Check that the +pretend output we sent ends up in the current build log. + + WHEN worker manager calls POST /worker/bar/snippet, + ... with JSON body '{ + ... "stdout": "ikiwiki build output", + ... "stderr": "", + ... "exit-code": null + ... }' + AND user calls GET /projects/foo/logs/current + THEN response has status 200, and text body "ikiwiki build output" + +The current build step hasn't changed. + + WHEN worker manager calls GET /worker/bar + THEN response has status 200, + ... and JSON body "{ + ... "project": "foo", + ... "shell": "ikiwiki --build" + ... }" + +Pretend current command finishes. Make sure current log updates, and +that we get a new thing to run. + + WHEN worker manager calls POST /worker/bar/snippet, + ... with JSON body '{ + ... "stdout": "|more output", + ... "stderr": "", + ... "exit-code": 0 + ... }' + AND user calls GET /projects/foo/logs/current + THEN response has status 200, and an empty body + + WHEN user calls GET /projects/foo/logs/previous + THEN response has status 200, + ... and text body "ikiwiki build output|more output" + + WHEN worker manager calls GET /worker/bar + THEN response has status 200, + ... and JSON body "{ + ... "project": "foo", + ... "shell": "rsync" + ... }" + +Tell worker the rsync command also finishes. After that, there should +be nothing more to do. The current log should become empty, the +previous log will contain the previously current log. + + WHEN worker manager calls POST /worker/bar/snippet, + ... with JSON body '{ + ... "stdout": "rsync output", + ... "stderr": "", + ... "exit-code": 0 + ... }' + + WHEN user calls GET /projects/foo/logs/current + THEN response has status 200, and an empty body + + WHEN user calls GET /projects/foo/logs/previous + THEN response has status 200, and text body "rsync output" + + WHEN user calls GET /worker/bar + THEN response has status 200, and an empty body + +And we're done. + + FINALLY stop controller instance diff --git a/yarns/900.yarn b/yarns/900.yarn index 41711bb..4ac722b 100644 --- a/yarns/900.yarn +++ b/yarns/900.yarn @@ -1,5 +1,7 @@ # Scenario step implementations +# Manage controller instance + IMPLEMENTS GIVEN a running controller instance controller = os.path.join(srcdir, 'controller') cliapp.runcmd(['/usr/sbin/daemonize', '-c.', controller, 'pid', 'port']) @@ -11,26 +13,72 @@ print 'killing process', repr(vars['pid']) os.kill(int(vars['pid']), signal.SIGTERM) - IMPLEMENTS WHEN client makes request GET (\S+) - path = os.environ['MATCH_1'] - url = 'http://localhost:{}{}'.format(vars['port'], path) - print 'url:', repr(url) - import requests - r = requests.get(url) - vars['http-status'] = r.status_code - vars['http-body'] = r.text - - IMPLEMENTS THEN HTTP status code is (\d+) - wanted = int(os.environ['MATCH_1']) - print 'wanted:', repr(wanted) - print 'actual:', repr(vars['http-status']) - assert vars['http-status'] == wanted - - IMPLEMENTS THEN result matches (.*) - import json - body = json.loads(vars['http-body']) - pattern = json.loads(os.environ['MATCH_1']) - assert type(body) == type(pattern) - for key in pattern.keys(): - assert key in body - assert body[key] == pattern[key] + +## Git repositories + + IMPLEMENTS GIVEN a git repo (\S+) with file (\S+) containing "(.+)" + repo = yarnutils.get_next_match() + filename = yarnutils.get_next_match() + content = yarnutils.get_next_match() + pathname = os.path.join(repo, filename) + os.mkdir(repo) + git(repo, 'init', '.') + write(pathname, unescape(content)) + git(repo, 'add', '.') + git(repo, 'commit', '-minitial') + +## Controller configuration + + IMPLEMENTS GIVEN a project (\S+), using (\S+), publishing to (\S+) + project = yarnutils.get_next_match() + repo = yarnutils.get_next_match() + server = yarnutils.get_next_match() + config = { + project: { + 'git': repo, + 'shell': 'foo', + } + } + write('ick.ick', yaml.safe_dump(config)) + +## API use + + IMPLEMENTS WHEN (user|worker manager|git server) calls (\S+) (\S+) + who = yarnutils.get_next_match() + method = yarnutils.get_next_match() + path = yarnutils.get_next_match() + url = controller_url(vars['port'], path) + vars['status'], vars['body'] = request(method, url) + + IMPLEMENTS WHEN worker manager calls (\S+) (\S+), with JSON body '(.+)' + method = yarnutils.get_next_match() + path = yarnutils.get_next_match() + body_text = yarnutils.get_next_match() + url = controller_url(vars['port'], path) + body = parse_json(body_text) + vars['status'], vars['body'] = request(method, url, body=body_text) + + IMPLEMENTS THEN response has status (\d+), and an empty body + status = yarnutils.get_next_match() + yarnutils.assertEqual(int(status), int(vars['status'])) + yarnutils.assertEqual(vars['body'], '') + + IMPLEMENTS THEN response has status (\d+), and text body "(.+)" + status = yarnutils.get_next_match() + bodypat = yarnutils.get_next_match() + yarnutils.assertEqual(int(status), int(vars['status'])) + yarnutils.assertEqual(vars['body'], unescape(bodypat) ) + + IMPLEMENTS THEN response has status (\d+), and JSON body "(.+)" + status = yarnutils.get_next_match() + bodytext = yarnutils.get_next_match() + print 'varsbody:', repr(vars['body']) + print 'bodytext:', repr(bodytext) + bodyjson = parse_json(bodytext) + print 'bodyjson:', repr(bodyjson) + yarnutils.assertEqual(int(status), int(vars['status'])) + yarnutils.assertEqual(parse_json(vars['body']), parse_json(bodytext) ) + + IMPLEMENTS THEN response has status (\d+) + status = yarnutils.get_next_match() + yarnutils.assertEqual(int(status), int(vars['status'])) diff --git a/yarns/lib.py b/yarns/lib.py index b7df3a7..9040bb7 100644 --- a/yarns/lib.py +++ b/yarns/lib.py @@ -1,11 +1,15 @@ import errno +import json import os +import StringIO import time import cliapp - +import requests +import yaml import yarnutils + datadir = os.environ['DATADIR'] srcdir = os.environ['SRCDIR'] @@ -27,3 +31,60 @@ def cat(filename): continue raise raise Exception("cat took more then %s seconds" % MAX_CAT_TIME) + + +def write(filename, content): + with open(filename, 'w') as f: + f.write(content) + + +def git(repo, *argv): + return cliapp.runcmd(['git'] + list(argv), cwd=repo) + + +def controller_url(port, path): + return 'http://localhost:{}{}'.format(port, path) + + +def request(method, url, body=None): + funcs = { + 'POST': requests.post, + 'PUT': requests.put, + 'GET': requests.get, + 'DELETE': requests.delete, + } + + headers = { + 'Content-Type': 'application/json', + } + + response = funcs[method]( + url, + headers=headers, + data=body, + ) + + return response.status_code, response.text + + +def parse_json(text): + return json.loads(text) + + +def parse_yaml(text): + f = StringIO.StringIO(text) + return yaml.safe_load(stream=f) + + +def unescape(text): + def helper(text): + while text: + if text.startswith('\\n'): + skip = 2 + answer = '\n' + else: + skip = 1 + answer = text[0] + text = text[skip:] + yield answer + return ''.join(helper(text)) |