From 888db73b93aefe70d838d499f7f9cc43eee7372b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 6 Aug 2017 11:37:46 +0300 Subject: Start rewrite using Python 3, apifw, slog apifw and slog are two libraries I've written for work. They make writing RESTful HTTP JSON APIs easier. --- README | 4 + controller | 7 -- ick2-controller | 38 ------ ick2lib/__init__.py | 22 ---- ick2lib/apiservice.py | 163 ------------------------- ick2lib/app.py | 99 --------------- ick2lib/project.py | 69 ----------- ick2lib/project_tests.py | 79 ------------ ick2lib/version.py | 2 - run-debug | 12 -- uwsgi.ini | 12 -- yarns/000.yarn | 24 ---- yarns/100-hello.yarn | 15 --- yarns/200-build.yarn | 305 ----------------------------------------------- yarns/900.yarn | 96 --------------- yarns/lib.py | 111 ----------------- 16 files changed, 4 insertions(+), 1054 deletions(-) delete mode 100755 controller delete mode 100644 ick2-controller delete mode 100644 ick2lib/__init__.py delete mode 100644 ick2lib/apiservice.py delete mode 100644 ick2lib/app.py delete mode 100644 ick2lib/project.py delete mode 100644 ick2lib/project_tests.py delete mode 100644 ick2lib/version.py delete mode 100755 run-debug delete mode 100644 uwsgi.ini delete mode 100644 yarns/000.yarn delete mode 100644 yarns/100-hello.yarn delete mode 100644 yarns/200-build.yarn delete mode 100644 yarns/900.yarn delete mode 100644 yarns/lib.py diff --git a/README b/README index 5fba99c..b604494 100644 --- a/README +++ b/README @@ -1,6 +1,10 @@ Ick - a continuous integration system ============================================================================= +FIXME: this software is being rewritten to use gunicorn instead of +uwsgi and apifw, a little framework I wrote for work for RESTful HTTP +APIs. Also, Python3. + Introduction ----------------------------------------------------------------------------- diff --git a/controller b/controller deleted file mode 100755 index 66522c9..0000000 --- a/controller +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python2 - - -import ick2lib - - -ick2lib.ApiApp().run() diff --git a/ick2-controller b/ick2-controller deleted file mode 100644 index 91b593b..0000000 --- a/ick2-controller +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python2 - - -import argparse -import logging -import sys - -import yaml - -import ick2lib - - -def load_projects(filename): - with open(filename) as f: - projects_config = yaml.safe_load(f) - - projects = {} - for name, config in projects_config['projects'].items(): - p = ick2lib.Project(name) - projects[name] = p - for shell in config['shell_steps']: - p.add_build_step(shell) - - return projects - - -parser = argparse.ArgumentParser() -parser.add_argument('--projects', action='store', dest='projects') -parser.add_argument('--log', action='store', dest='log', default='/dev/null') -results = parser.parse_args() - -logging.basicConfig(filename=results.log, level=logging.DEBUG) - -logging.info('ick2-controller starts') -projects = load_projects(results.projects) -service = ick2lib.ApiService() -service.set_projects(projects) -application = service.get_uwsgi_app() diff --git a/ick2lib/__init__.py b/ick2lib/__init__.py deleted file mode 100644 index a195273..0000000 --- a/ick2lib/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# =*= License: GPL-3+ =*= - - -from .version import __version__, __version_info__ -from .project import Project -from .apiservice import ApiService -from .app import ApiApp diff --git a/ick2lib/apiservice.py b/ick2lib/apiservice.py deleted file mode 100644 index 2120391..0000000 --- a/ick2lib/apiservice.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# =*= License: GPL-3+ =*= - - -import logging -import time - - -import bottle - - -import ick2lib - - -class ApiService(object): - - def __init__(self): - self._projects = {} - self._app = self._create_bottle_app() - - def _create_bottle_app(self): - app = bottle.Bottle() - - routes = [ - { - 'method': 'GET', - 'path': '/', - 'callback': lambda: None, - }, - { - 'method': 'GET', - 'path': '/version', - 'callback': lambda: { 'version': '1.0' }, - }, - { - 'method': 'GET', - 'path': '/projects', - 'callback': self._projects_cb, - }, - { - 'method': 'GET', - 'path': '/projects//logs/current', - 'callback': self._current_log_cb, - }, - { - 'method': 'GET', - 'path': '/projects//logs/previous', - 'callback': self._previous_log_cb, - }, - { - 'method': 'GET', - 'path': '/projects//+trigger', - 'callback': self._trigger_cb, - }, - { - 'method': 'GET', - 'path': '/worker/', - 'callback': self._work_cb, - }, - { - 'method': 'POST', - 'path': '/worker//snippet', - 'callback': self._snippet_cb, - }, - ] - - for route in routes: - app.route(**route) - - app.install(LoggingPlugin()) - return app - - def _projects_cb(self): - return { - 'projects': list(sorted(self._projects.keys())), - } - - def _current_log_cb(self, project): - return self._projects[project].get_current_log() - - def _previous_log_cb(self, project): - return self._projects[project].get_previous_log() - - def _trigger_cb(self, project): - return self._projects[project].trigger_build() - - def _work_cb(self, worker): - for p in self._projects.values(): - step = p.get_current_build_step(worker) - if step is not None: - return { - 'project': p.name, - 'git': p.git, - 'shell': step, - } - return None - - def _snippet_cb(self, worker): - obj = bottle.request.json - p = self._projects[obj['project']] - logging.debug('stdout: %r', obj['stdout']) - logging.debug('stderr: %r', obj['stderr']) - p.append_to_current_log(obj['stdout'] + obj['stderr']) - logging.debug('log is now: %r', p.get_current_log()) - if obj['exit-code'] is not None: - p.finish_current_log() - p.finish_current_build_step() - - def set_projects(self, projects): - self._projects = projects - - def get_uwsgi_app(self): - return self._app - - def run_debug(self, port): - self._app.run(port=port, quiet=True) - - - -class LoggingPlugin(object): - - def apply(self, callback, route): - - def wrapper(*args, **kwargs): - # Do the thing and catch any exceptions. - try: - self.log_request() - data = callback(*args, **kwargs) - self.add_response_headers() - self.log_response() - return data - except SystemExit: - raise - except BaseException as e: - logging.error(str(e), exc_info=True) - raise bottle.HTTPError(status=500, body=str(e)) - - return wrapper - - def add_response_headers(self): - rfc822 = time.strftime('%a, %d %b %Y %H:%M:%S %z') - bottle.response.set_header('Date', rfc822) - - def log_request(self): - r = bottle.request - logging.info('Request: %s %s', r.method, r.path) - - def log_response(self): - logging.info('Response: %s', bottle.response.status_code) diff --git a/ick2lib/app.py b/ick2lib/app.py deleted file mode 100644 index 1f7cb25..0000000 --- a/ick2lib/app.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# =*= License: GPL-3+ =*= - - -import logging -import os -import random - - -import cliapp -import yaml - - -import ick2lib - - -class ApiApp(cliapp.Application): - - def add_settings(self): - self.settings.boolean( - ['debug'], - 'Turn on debug mode (incl. for yarn tests)' - ) - - self.settings.string( - ['pid-file'], - 'Store application PID in FILE', - metavar='FILE' - ) - - self.settings.string( - ['port-file'], - 'Store randomly chosen listening port (in debug mode) in FILE', - metavar='FILE' - ) - - self.settings.string( - ['projects'], - 'Read project list from YAML format FILE', - metavar='FILE' - ) - - def write_pid_file(self): - write_file(self.settings['pid-file'], str(os.getpid())) - - def pick_random_port(self): - return random.randint(1025, 32767) - - def write_port_file(self, port): - write_file(self.settings['port-file'], str(port)) - - def load_projects(self, filename): - with open(filename) as f: - projects_config = yaml.safe_load(f) - - projects = {} - for name, config in projects_config['projects'].items(): - p = ick2lib.Project(name) - p.set_git(config['git']) - projects[name] = p - for shell in config['shell_steps']: - p.add_build_step(shell) - - return projects - - def process_args(self, args): - assert self.settings['debug'] - - projects = {} - if os.path.exists(self.settings['projects']): - projects = self.load_projects(self.settings['projects']) - - self.write_pid_file() - port = self.pick_random_port() - self.write_port_file(port) - - apiapp = ick2lib.ApiService() - apiapp.set_projects(projects) - apiapp.run_debug(port) - - -def write_file(filename, content): - logging.info('Writing file %s: %r', filename, content) - with open(filename, 'w') as f: - f.write('{}\n'.format(content)) diff --git a/ick2lib/project.py b/ick2lib/project.py deleted file mode 100644 index ae9a124..0000000 --- a/ick2lib/project.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# =*= License: GPL-3+ =*= - - -class Project(object): - - def __init__(self, name): - self.name = name - self.git = None - self.current_log = '' - self.previous_log = '' - self.build_steps = [] - self.current_build_step = None - self.current_worker = None - - def set_git(self, url): - self.git = url - - def get_current_log(self): - return self.current_log - - def get_previous_log(self): - return self.previous_log - - def append_to_current_log(self, text): - self.current_log += text - - def finish_current_log(self): - self.previous_log = self.current_log - self.current_log = '' - - def get_build_steps(self): - return self.build_steps - - def add_build_step(self, shell_text): - self.build_steps.append(shell_text) - - def get_current_build_step(self, worker_name): - if self.current_build_step is None: - return None - if self.current_worker is None: - self.current_worker = worker_name - elif self.current_worker != worker_name: - return None - return self.build_steps[self.current_build_step] - - def finish_current_build_step(self): - assert self.current_build_step is not None - self.current_build_step += 1 - if self.current_build_step >= len(self.build_steps): - self.current_build_step = None - - def trigger_build(self): - if self.current_build_step is None: - self.current_build_step = 0 diff --git a/ick2lib/project_tests.py b/ick2lib/project_tests.py deleted file mode 100644 index 8e9f796..0000000 --- a/ick2lib/project_tests.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# =*= License: GPL-3+ =*= - - -import unittest - -import ick2lib - - -class ProjectTests(unittest.TestCase): - - def test_has_name(self): - p = ick2lib.Project('foo') - self.assertEqual(p.name, 'foo') - - def test_has_git(self): - p = ick2lib.Project('foo') - p.set_git('git://foo') - self.assertEqual(p.git, 'git://foo') - - def test_has_current_log_initially_empty(self): - p = ick2lib.Project('foo') - self.assertEqual(p.get_current_log(), '') - - def test_appends_to_current_log(self): - p = ick2lib.Project('foo') - p.append_to_current_log('output') - self.assertEqual(p.get_current_log(), 'output') - - def test_finished_current_log(self): - p = ick2lib.Project('foo') - p.append_to_current_log('output') - p.finish_current_log() - self.assertEqual(p.get_current_log(), '') - self.assertEqual(p.get_previous_log(), 'output') - - def test_has_previous_log_initially_empty(self): - p = ick2lib.Project('foo') - self.assertEqual(p.get_previous_log(), '') - - def test_has_no_build_steps_initially(self): - p = ick2lib.Project('foo') - self.assertEqual(p.get_build_steps(), []) - - def test_adds_build_steps(self): - p = ick2lib.Project('foo') - p.add_build_step('echo') - self.assertEqual(p.get_build_steps(), ['echo']) - - def test_tracks_current_build_step(self): - p = ick2lib.Project('foo') - p.add_build_step('step1') - p.add_build_step('step2') - self.assertEqual(p.get_current_build_step('worker1'), None) - self.assertEqual(p.get_current_build_step('worker2'), None) - p.trigger_build() - self.assertEqual(p.get_current_build_step('worker1'), 'step1') - self.assertEqual(p.get_current_build_step('worker1'), 'step1') - self.assertEqual(p.get_current_build_step('worker2'), None) - p.finish_current_build_step() - self.assertEqual(p.get_current_build_step('worker1'), 'step2') - self.assertEqual(p.get_current_build_step('worker2'), None) - p.finish_current_build_step() - self.assertEqual(p.get_current_build_step('worker1'), None) - self.assertEqual(p.get_current_build_step('worker2'), None) diff --git a/ick2lib/version.py b/ick2lib/version.py deleted file mode 100644 index 20059f9..0000000 --- a/ick2lib/version.py +++ /dev/null @@ -1,2 +0,0 @@ -__version__ = "0.2.1+git" -__version_info__ = (0, 2, 1, '+git') diff --git a/run-debug b/run-debug deleted file mode 100755 index a960e6a..0000000 --- a/run-debug +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -set -eu - -ick="$1" - -./controller \ - --debug \ - --pid-file=pid \ - --port-file=port \ - --log=log \ - --projects="$ick" \ diff --git a/uwsgi.ini b/uwsgi.ini deleted file mode 100644 index 2a427c4..0000000 --- a/uwsgi.ini +++ /dev/null @@ -1,12 +0,0 @@ -[uwsgi] -logger = python -http-socket = 0.0.0.0:12765 -master = true -plugins = python -wsgi-file = /usr/bin/ick2-controller -pyargv=--projects /etc/ick2/projects.yaml --log /var/log/ick2/ick2.log -uid = www-data -gid = www-data -workers = 1 -threads = 1 -buffer-size = 32768 diff --git a/yarns/000.yarn b/yarns/000.yarn deleted file mode 100644 index 7a19824..0000000 --- a/yarns/000.yarn +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: Ick2 controller yarns -... - - -Introduction -============================================================================= - -Ick2 will be a continuous integration system. Its core component is -the **controller**, which does nothing, except decides what workers should -do. It knows of the projects that can be built, and keeps track what -step is being run on each project, and collects build output from the -workers. - -This document specifies the yarn test scenarios for a minimal viable -version of the controller: there will be a small number of project, -each project builds a web site from source in git, using ikiwiki, and -publishes the website on a server using rsync. - -The controller provides an HTTP API for controlling a build. The API -is used by an external entity (such as the git server) to trigger a -build, and by workers or worker proxies to request something to do, -and to report results. The API may also eventually be used by end -users to query status and results. diff --git a/yarns/100-hello.yarn b/yarns/100-hello.yarn deleted file mode 100644 index d719497..0000000 --- a/yarns/100-hello.yarn +++ /dev/null @@ -1,15 +0,0 @@ -# Controller smoke test - -This scenario is just for making sure we can, in our tests, start and -stop the controller, and make requests to it. - - SCENARIO controller smoke test - GIVEN a running controller instance - - WHEN user calls GET /version - THEN response has status 200, and JSON body "{ "version": "1.0" }" - - WHEN user calls GET /blatherskite - THEN response has status 404 - - FINALLY stop controller instance diff --git a/yarns/200-build.yarn b/yarns/200-build.yarn deleted file mode 100644 index 3465f50..0000000 --- a/yarns/200-build.yarn +++ /dev/null @@ -1,305 +0,0 @@ -Building a project -============================================================================= - -One build, one worker ------------------------------------------------------------------------------ - -This section uses the controller to walk through all the steps for a -build. We start with some setup, defining a git repo and an ick -project and starting the controller. - - SCENARIO run a build with one worker - - GIVEN a git repo foo.git with file index.mdwn containing - ... "hello, world\n" - AND a project foo, using foo.git, publishing to foo-web - AND a running controller instance - -Ensure controller knows of the project. - - WHEN user calls GET /projects - THEN response has status 200, - ... and JSON body "{ "projects": [ "foo" ] }" - -There is no job running now, so if the worker manager asks for work, -it gets nothing. - - WHEN worker manager calls GET /worker/bar - THEN response has status 200, and an empty body - -Trigger a new build. There is now work to do. - - WHEN git server calls GET /projects/foo/+trigger - THEN response has status 200 - - WHEN worker manager calls GET /worker/bar - THEN response has status 200, and JSON body "{ - ... "project": "foo", - ... "git": "foo.git", - ... "shell": "ikiwiki --build" - ... }" - -Pretend a job is running, and send output to the controller. Don't send -an exit code, since the pretend job hasn't finished. Check that the -pretend output we sent ends up in the current build log. - - WHEN worker manager calls POST /worker/bar/snippet, - ... with JSON body '{ - ... "project": "foo", - ... "stdout": "ikiwiki build output", - ... "stderr": "", - ... "exit-code": null - ... }' - AND user calls GET /projects/foo/logs/current - THEN response has status 200, and text body "ikiwiki build output" - -The current build step hasn't changed. - - WHEN worker manager calls GET /worker/bar - THEN response has status 200, - ... and JSON body "{ - ... "project": "foo", - ... "git": "foo.git", - ... "shell": "ikiwiki --build" - ... }" - -Pretend current command finishes. Make sure current log updates, and -that we get a new thing to run. - - WHEN worker manager calls POST /worker/bar/snippet, - ... with JSON body '{ - ... "project": "foo", - ... "stdout": "|more output", - ... "stderr": "", - ... "exit-code": 0 - ... }' - AND user calls GET /projects/foo/logs/current - THEN response has status 200, and an empty body - - WHEN user calls GET /projects/foo/logs/previous - THEN response has status 200, - ... and text body "ikiwiki build output|more output" - - WHEN worker manager calls GET /worker/bar - THEN response has status 200, - ... and JSON body "{ - ... "project": "foo", - ... "git": "foo.git", - ... "shell": "rsync" - ... }" - -Tell worker the rsync command also finishes. After that, there should -be nothing more to do. The current log should become empty, the -previous log will contain the previously current log. - - WHEN worker manager calls POST /worker/bar/snippet, - ... with JSON body '{ - ... "project": "foo", - ... "stdout": "rsync output", - ... "stderr": "", - ... "exit-code": 0 - ... }' - - WHEN user calls GET /projects/foo/logs/current - THEN response has status 200, and an empty body - - WHEN user calls GET /projects/foo/logs/previous - THEN response has status 200, and text body "rsync output" - - WHEN user calls GET /worker/bar - THEN response has status 200, and an empty body - -And we're done. - - FINALLY stop controller instance - - -Two builds, two workers ------------------------------------------------------------------------------ - -This section runs two builds using two workers. The primary goal here -is to make sure each worker gets consecutive steps for its own build, -so that it runs all the steps in the same workspace. - -We start with some setup, defining a git repo and an ick project and -starting the controller. - - SCENARIO run two builds with two workers - - GIVEN a git repo foo.git with file index.mdwn containing - ... "hello, world\n" - AND a git repo bar.git with file index.mdwn containing - ... "hello, bar\n" - AND a project foo, using foo.git, publishing to foo-web - AND a project bar, using bar.git, publishing to bar-web - AND a running controller instance - -Ensure controller knows of both projects. The list of project names -should be sorted alphabetically. - - WHEN user calls GET /projects - THEN response has status 200, - ... and JSON body "{ "projects": [ "bar", "foo" ] }" - -There is no job running now, so if the worker manager asks for work, -it gets nothing. - - WHEN worker manager calls GET /worker/one - THEN response has status 200, and an empty body - -Trigger new builds on both projects. There is now work to do. - - WHEN git server calls GET /projects/foo/+trigger - THEN response has status 200 - - WHEN worker manager calls GET /worker/one - THEN response has status 200, and JSON body "{ - ... "project": "foo", - ... "git": "foo.git", - ... "shell": "ikiwiki --build" - ... }" - -The second worker should still get nothing to do. - - WHEN worker manager calls GET /worker/two - THEN response has status 200, and an empty body - -Trigger the other project, and the second worker gets something to do. - - WHEN git server calls GET /projects/bar/+trigger - THEN response has status 200 - - WHEN worker manager calls GET /worker/two - THEN response has status 200, and JSON body "{ - ... "project": "bar", - ... "git": "bar.git", - ... "shell": "ikiwiki --build" - ... }" - -Pretend the build step for the first project is running, and send -output to the controller. Don't send an exit code, since the pretend -step hasn't finished. Check that the pretend output we sent ends up in -the current build log. - - WHEN worker manager calls POST /worker/one/snippet, - ... with JSON body '{ - ... "project": "foo", - ... "stdout": "ikiwiki build output", - ... "stderr": "", - ... "exit-code": null - ... }' - AND user calls GET /projects/foo/logs/current - THEN response has status 200, and text body "ikiwiki build output" - -The current build step hasn't changed. - - WHEN worker manager calls GET /worker/one - THEN response has status 200, - ... and JSON body "{ - ... "project": "foo", - ... "git": "foo.git", - ... "shell": "ikiwiki --build" - ... }" - -Pretend the build step finishes. Make sure current log updates, and -that we get a new thing to run. - - WHEN worker manager calls POST /worker/one/snippet, - ... with JSON body '{ - ... "project": "foo", - ... "stdout": "|more output", - ... "stderr": "", - ... "exit-code": 0 - ... }' - AND user calls GET /projects/foo/logs/current - THEN response has status 200, and an empty body - - WHEN user calls GET /projects/foo/logs/previous - THEN response has status 200, - ... and text body "ikiwiki build output|more output" - - WHEN worker manager calls GET /worker/one - THEN response has status 200, - ... and JSON body "{ - ... "project": "foo", - ... "git": "foo.git", - ... "shell": "rsync" - ... }" - -The other worker is still running its step, and if it asks, it gets -the same build step to run. - - WHEN worker manager calls GET /worker/two - THEN response has status 200, and JSON body "{ - ... "project": "bar", - ... "git": "bar.git", - ... "shell": "ikiwiki --build" - ... }" - -Tell controller the rsync command of the first project also finishes. -After that, there should be nothing more to do. The current log should -become empty, the previous log will contain the previously current -log. - - WHEN worker manager calls POST /worker/one/snippet, - ... with JSON body '{ - ... "project": "foo", - ... "stdout": "rsync output", - ... "stderr": "", - ... "exit-code": 0 - ... }' - - WHEN user calls GET /projects/foo/logs/current - THEN response has status 200, and an empty body - - WHEN user calls GET /projects/foo/logs/previous - THEN response has status 200, and text body "rsync output" - - WHEN user calls GET /worker/one - THEN response has status 200, and an empty body - -Finish the other project build. - - WHEN worker manager calls POST /worker/two/snippet, - ... with JSON body '{ - ... "project": "bar", - ... "stdout": "ikiwiki output", - ... "stderr": "", - ... "exit-code": 0 - ... }' - AND user calls GET /projects/bar/logs/current - THEN response has status 200, and an empty body - - WHEN user calls GET /projects/bar/logs/previous - THEN response has status 200, - ... and text body "ikiwiki output" - - WHEN worker manager calls GET /worker/two - THEN response has status 200, - ... and JSON body "{ - ... "project": "bar", - ... "git": "bar.git", - ... "shell": "rsync" - ... }" - - WHEN worker manager calls POST /worker/two/snippet, - ... with JSON body '{ - ... "project": "bar", - ... "stdout": "second worker rsync output", - ... "stderr": "", - ... "exit-code": 0 - ... }' - - WHEN user calls GET /projects/bar/logs/current - THEN response has status 200, and an empty body - - WHEN user calls GET /projects/bar/logs/previous - THEN response has status 200, and text body - ... "second worker rsync output" - - WHEN user calls GET /worker/two - THEN response has status 200, and an empty body - -And we're done. - - FINALLY stop controller instance diff --git a/yarns/900.yarn b/yarns/900.yarn deleted file mode 100644 index 8ee31f3..0000000 --- a/yarns/900.yarn +++ /dev/null @@ -1,96 +0,0 @@ -# Scenario step implementations - -# Manage controller instance - - IMPLEMENTS GIVEN a running controller instance - controller = os.path.join(srcdir, 'controller') - cliapp.runcmd( - ['/usr/sbin/daemonize', '-c.', - controller, '--debug', '--pid-file=pid', - '--port-file=port', '--projects=ick.ick', '--log=log']) - vars['pid'] = cat('pid').strip() - vars['port'] = cat('port').strip() - - IMPLEMENTS FINALLY stop controller instance - import signal - print 'killing process', repr(vars['pid']) - os.kill(int(vars['pid']), signal.SIGTERM) - - -## Git repositories - - IMPLEMENTS GIVEN a git repo (\S+) with file (\S+) containing "(.+)" - repo = yarnutils.get_next_match() - filename = yarnutils.get_next_match() - content = yarnutils.get_next_match() - pathname = os.path.join(repo, filename) - os.mkdir(repo) - git(repo, 'init', '.') - git(repo, 'config', 'user.email', 'user@example.com') - git(repo, 'config', 'user.name', 'J. Random User') - write(pathname, unescape(content)) - git(repo, 'add', '.') - git(repo, 'commit', '-minitial') - -## Controller configuration - - IMPLEMENTS GIVEN a project (\S+), using (\S+), publishing to (\S+) - project = yarnutils.get_next_match() - repo = yarnutils.get_next_match() - - if os.path.exists('ick.ick'): - config = yaml.safe_load(open('ick.ick')) - else: - config = { - 'projects': {} - } - config['projects'][project] = { - 'git': repo, - 'shell_steps': [ - 'ikiwiki --build', - 'rsync', - ], - } - write('ick.ick', yaml.safe_dump(config)) - -## API use - - IMPLEMENTS WHEN (user|worker manager|git server) calls (\S+) (\S+) - who = yarnutils.get_next_match() - method = yarnutils.get_next_match() - path = yarnutils.get_next_match() - url = controller_url(vars['port'], path) - vars['status'], vars['body'] = request(method, url) - - IMPLEMENTS WHEN worker manager calls (\S+) (\S+), with JSON body '(.+)' - method = yarnutils.get_next_match() - path = yarnutils.get_next_match() - body_text = yarnutils.get_next_match() - url = controller_url(vars['port'], path) - body = parse_json(body_text) - vars['status'], vars['body'] = request(method, url, body=body_text) - - IMPLEMENTS THEN response has status (\d+), and an empty body - status = yarnutils.get_next_match() - yarnutils.assertEqual(int(status), int(vars['status'])) - yarnutils.assertEqual(vars['body'], '') - - IMPLEMENTS THEN response has status (\d+), and text body "(.+)" - status = yarnutils.get_next_match() - bodypat = yarnutils.get_next_match() - yarnutils.assertEqual(int(status), int(vars['status'])) - yarnutils.assertEqual(vars['body'], unescape(bodypat) ) - - IMPLEMENTS THEN response has status (\d+), and JSON body "(.+)" - status = yarnutils.get_next_match() - bodytext = yarnutils.get_next_match() - print 'varsbody:', repr(vars['body']) - print 'bodytext:', repr(bodytext) - bodyjson = parse_json(bodytext) - print 'bodyjson:', repr(bodyjson) - yarnutils.assertEqual(int(status), int(vars['status'])) - yarnutils.assertEqual(parse_json(vars['body']), parse_json(bodytext) ) - - IMPLEMENTS THEN response has status (\d+) - status = yarnutils.get_next_match() - yarnutils.assertEqual(int(status), int(vars['status'])) diff --git a/yarns/lib.py b/yarns/lib.py deleted file mode 100644 index fc4acfc..0000000 --- a/yarns/lib.py +++ /dev/null @@ -1,111 +0,0 @@ -import errno -import json -import os -import StringIO -import time - -import cliapp -import requests -import yaml -import yarnutils - - -datadir = os.environ['DATADIR'] -srcdir = os.environ['SRCDIR'] - -vars = yarnutils.Variables(datadir) - - -MAX_CAT_TIME = 5 # seconds -def cat(filename): - start = time.time() - while time.time() - start < MAX_CAT_TIME: - try: - with open(filename) as f: - data = f.read() - if len(data) == 0: - continue - return data - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - continue - raise - raise Exception("cat took more then %s seconds" % MAX_CAT_TIME) - - -def write(filename, content): - with open(filename, 'w') as f: - f.write(content) - - -def git(repo, *argv): - return cliapp.runcmd(['git'] + list(argv), cwd=repo) - - -def controller_url(port, path): - return 'http://localhost:{}{}'.format(port, path) - - -def request(method, url, body=None): - funcs = { - 'POST': requests.post, - 'PUT': requests.put, - 'GET': requests.get, - 'DELETE': requests.delete, - } - - headers = { - 'Content-Type': 'application/json', - } - - response = funcs[method]( - url, - headers=headers, - data=body, - ) - - return response.status_code, response.text - - -def parse_json(text): - return json.loads(text, object_pairs_hook=dictify) - - - -def dictify(pairs): - return { - stringify(key): stringify(value) - for key, value in pairs - } - - -def stringify(x): - if isinstance(x, unicode): - return str(x) - if isinstance(x, list): - return [stringify(y) for y in x] - if isinstance(x, dict): - return { - stringify(key): stringify(value) - for key, value in pairs - } - return x - - -def parse_yaml(text): - f = StringIO.StringIO(text) - return yaml.safe_load(stream=f) - - -def unescape(text): - def helper(text): - while text: - if text.startswith('\\n'): - skip = 2 - answer = '\n' - else: - skip = 1 - answer = text[0] - text = text[skip:] - yield answer - return ''.join(helper(text)) -- cgit v1.2.1