Start rewrite using Python 3, apifw, slog

apifw and slog are two libraries I've written for work. They make
writing RESTful HTTP JSON APIs easier.

16 files changed, 4 insertions, 1054 deletions

diff --git a/README b/README
index 5fba99c..b604494 100644
--- a/README
+++ b/README
@@ -1,6 +1,10 @@
 Ick - a continuous integration system
 =============================================================================
 
+FIXME: this software is being rewritten to use gunicorn instead of
+uwsgi and apifw, a little framework I wrote for work for RESTful HTTP
+APIs. Also, Python3.
+
 Introduction
 -----------------------------------------------------------------------------
 
diff --git a/controller b/controller
deleted file mode 100755
index 66522c9..0000000
--- a/controller
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env python2
-
-
-import ick2lib
-
-
-ick2lib.ApiApp().run()
diff --git a/ick2-controller b/ick2-controller
deleted file mode 100644
index 91b593b..0000000
--- a/ick2-controller
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python2
-
-
-import argparse
-import logging
-import sys
-
-import yaml
-
-import ick2lib
-
-
-def load_projects(filename):
-    with open(filename) as f:
-        projects_config = yaml.safe_load(f)
-
-    projects = {}
-    for name, config in projects_config['projects'].items():
-        p = ick2lib.Project(name)
-        projects[name] = p
-        for shell in config['shell_steps']:
-            p.add_build_step(shell)
-
-    return projects
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument('--projects', action='store', dest='projects')
-parser.add_argument('--log', action='store', dest='log', default='/dev/null')
-results = parser.parse_args()
-
-logging.basicConfig(filename=results.log, level=logging.DEBUG)
-
-logging.info('ick2-controller starts')
-projects = load_projects(results.projects)
-service = ick2lib.ApiService()
-service.set_projects(projects)
-application = service.get_uwsgi_app()
diff --git a/ick2lib/__init__.py b/ick2lib/__init__.py
deleted file mode 100644
index a195273..0000000
--- a/ick2lib/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright 2017  Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# =*= License: GPL-3+ =*=
-
-
-from .version import __version__, __version_info__
-from .project import Project
-from .apiservice import ApiService
-from .app import ApiApp
diff --git a/ick2lib/apiservice.py b/ick2lib/apiservice.py
deleted file mode 100644
index 2120391..0000000
--- a/ick2lib/apiservice.py
+++ /dev/null
@@ -1,163 +0,0 @@
-# Copyright 2017  Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# =*= License: GPL-3+ =*=
-
-
-import logging
-import time
-
-
-import bottle
-
-
-import ick2lib
-
-
-class ApiService(object):
-
-    def __init__(self):
-        self._projects = {}
-        self._app = self._create_bottle_app()
-
-    def _create_bottle_app(self):
-        app = bottle.Bottle()
-
-        routes = [
-            {
-                'method': 'GET',
-                'path': '/',
-                'callback': lambda: None,
-            },
-            {
-                'method': 'GET',
-                'path': '/version',
-                'callback': lambda: { 'version': '1.0' },
-            },
-            {
-                'method': 'GET',
-                'path': '/projects',
-                'callback': self._projects_cb,
-            },
-            {
-                'method': 'GET',
-                'path': '/projects/<project>/logs/current',
-                'callback': self._current_log_cb,
-            },
-            {
-                'method': 'GET',
-                'path': '/projects/<project>/logs/previous',
-                'callback': self._previous_log_cb,
-            },
-            {
-                'method': 'GET',
-                'path': '/projects/<project>/+trigger',
-                'callback': self._trigger_cb,
-            },
-            {
-                'method': 'GET',
-                'path': '/worker/<worker>',
-                'callback': self._work_cb,
-            },
-            {
-                'method': 'POST',
-                'path': '/worker/<worker>/snippet',
-                'callback': self._snippet_cb,
-            },
-        ]
-
-        for route in routes:
-            app.route(**route)
-
-        app.install(LoggingPlugin())
-        return app
-
-    def _projects_cb(self):
-        return {
-            'projects': list(sorted(self._projects.keys())),
-        }
-
-    def _current_log_cb(self, project):
-        return self._projects[project].get_current_log()
-
-    def _previous_log_cb(self, project):
-        return self._projects[project].get_previous_log()
-
-    def _trigger_cb(self, project):
-        return self._projects[project].trigger_build()
-
-    def _work_cb(self, worker):
-        for p in self._projects.values():
-            step = p.get_current_build_step(worker)
-            if step is not None:
-                return {
-                    'project': p.name,
-                    'git': p.git,
-                    'shell': step,
-                }
-        return None
-
-    def _snippet_cb(self, worker):
-        obj = bottle.request.json
-        p = self._projects[obj['project']]
-        logging.debug('stdout: %r', obj['stdout'])
-        logging.debug('stderr: %r', obj['stderr'])
-        p.append_to_current_log(obj['stdout'] + obj['stderr'])
-        logging.debug('log is now: %r', p.get_current_log())
-        if obj['exit-code'] is not None:
-            p.finish_current_log()
-            p.finish_current_build_step()
-
-    def set_projects(self, projects):
-        self._projects = projects
-
-    def get_uwsgi_app(self):
-        return self._app
-
-    def run_debug(self, port):
-        self._app.run(port=port, quiet=True)
-
-
-
-class LoggingPlugin(object):
-
-    def apply(self, callback, route):
-
-        def wrapper(*args, **kwargs):
-            # Do the thing and catch any exceptions.
-            try:
-                self.log_request()
-                data = callback(*args, **kwargs)
-                self.add_response_headers()
-                self.log_response()
-                return data
-            except SystemExit:
-                raise
-            except BaseException as e:
-                logging.error(str(e), exc_info=True)
-                raise bottle.HTTPError(status=500, body=str(e))
-
-        return wrapper
-
-    def add_response_headers(self):
-        rfc822 = time.strftime('%a, %d %b %Y %H:%M:%S %z')
-        bottle.response.set_header('Date', rfc822)
-
-    def log_request(self):
-        r = bottle.request
-        logging.info('Request: %s %s', r.method, r.path)
-
-    def log_response(self):
-        logging.info('Response: %s', bottle.response.status_code)
diff --git a/ick2lib/app.py b/ick2lib/app.py
deleted file mode 100644
index 1f7cb25..0000000
--- a/ick2lib/app.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# Copyright 2017  Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# =*= License: GPL-3+ =*=
-
-
-import logging
-import os
-import random
-
-
-import cliapp
-import yaml
-
-
-import ick2lib
-
-
-class ApiApp(cliapp.Application):
-
-    def add_settings(self):
-        self.settings.boolean(
-            ['debug'],
-            'Turn on debug mode (incl. for yarn tests)'
-        )
-
-        self.settings.string(
-            ['pid-file'],
-            'Store application PID in FILE',
-            metavar='FILE'
-        )
-
-        self.settings.string(
-            ['port-file'],
-            'Store randomly chosen listening port (in debug mode) in FILE',
-            metavar='FILE'
-        )
-
-        self.settings.string(
-            ['projects'],
-            'Read project list from YAML format FILE',
-            metavar='FILE'
-        )
-
-    def write_pid_file(self):
-        write_file(self.settings['pid-file'], str(os.getpid()))
-
-    def pick_random_port(self):
-        return random.randint(1025, 32767)
-
-    def write_port_file(self, port):
-        write_file(self.settings['port-file'], str(port))
-
-    def load_projects(self, filename):
-        with open(filename) as f:
-            projects_config = yaml.safe_load(f)
-
-        projects = {}
-        for name, config in projects_config['projects'].items():
-            p = ick2lib.Project(name)
-            p.set_git(config['git'])
-            projects[name] = p
-            for shell in config['shell_steps']:
-                p.add_build_step(shell)
-
-        return projects
-
-    def process_args(self, args):
-        assert self.settings['debug']
-
-        projects = {}
-        if os.path.exists(self.settings['projects']):
-            projects = self.load_projects(self.settings['projects'])
-
-        self.write_pid_file()
-        port = self.pick_random_port()
-        self.write_port_file(port)
-
-        apiapp = ick2lib.ApiService()
-        apiapp.set_projects(projects)
-        apiapp.run_debug(port)
-
-
-def write_file(filename, content):
-    logging.info('Writing file %s: %r', filename, content)
-    with open(filename, 'w') as f:
-        f.write('{}\n'.format(content))
diff --git a/ick2lib/project.py b/ick2lib/project.py
deleted file mode 100644
index ae9a124..0000000
--- a/ick2lib/project.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright 2017  Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# =*= License: GPL-3+ =*=
-
-
-class Project(object):
-
-    def __init__(self, name):
-        self.name = name
-        self.git = None
-        self.current_log = ''
-        self.previous_log = ''
-        self.build_steps = []
-        self.current_build_step = None
-        self.current_worker = None
-
-    def set_git(self, url):
-        self.git = url
-
-    def get_current_log(self):
-        return self.current_log
-
-    def get_previous_log(self):
-        return self.previous_log
-
-    def append_to_current_log(self, text):
-        self.current_log += text
-
-    def finish_current_log(self):
-        self.previous_log = self.current_log
-        self.current_log = ''
-
-    def get_build_steps(self):
-        return self.build_steps
-
-    def add_build_step(self, shell_text):
-        self.build_steps.append(shell_text)
-
-    def get_current_build_step(self, worker_name):
-        if self.current_build_step is None:
-            return None
-        if self.current_worker is None:
-            self.current_worker = worker_name
-        elif self.current_worker != worker_name:
-            return None
-        return self.build_steps[self.current_build_step]
-
-    def finish_current_build_step(self):
-        assert self.current_build_step is not None
-        self.current_build_step += 1
-        if self.current_build_step >= len(self.build_steps):
-            self.current_build_step = None
-
-    def trigger_build(self):
-        if self.current_build_step is None:
-            self.current_build_step = 0
diff --git a/ick2lib/project_tests.py b/ick2lib/project_tests.py
deleted file mode 100644
index 8e9f796..0000000
--- a/ick2lib/project_tests.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# Copyright 2017  Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# =*= License: GPL-3+ =*=
-
-
-import unittest
-
-import ick2lib
-
-
-class ProjectTests(unittest.TestCase):
-
-    def test_has_name(self):
-        p = ick2lib.Project('foo')
-        self.assertEqual(p.name, 'foo')
-
-    def test_has_git(self):
-        p = ick2lib.Project('foo')
-        p.set_git('git://foo')
-        self.assertEqual(p.git, 'git://foo')
-
-    def test_has_current_log_initially_empty(self):
-        p = ick2lib.Project('foo')
-        self.assertEqual(p.get_current_log(), '')
-
-    def test_appends_to_current_log(self):
-        p = ick2lib.Project('foo')
-        p.append_to_current_log('output')
-        self.assertEqual(p.get_current_log(), 'output')
-
-    def test_finished_current_log(self):
-        p = ick2lib.Project('foo')
-        p.append_to_current_log('output')
-        p.finish_current_log()
-        self.assertEqual(p.get_current_log(), '')
-        self.assertEqual(p.get_previous_log(), 'output')
-
-    def test_has_previous_log_initially_empty(self):
-        p = ick2lib.Project('foo')
-        self.assertEqual(p.get_previous_log(), '')
-
-    def test_has_no_build_steps_initially(self):
-        p = ick2lib.Project('foo')
-        self.assertEqual(p.get_build_steps(), [])
-
-    def test_adds_build_steps(self):
-        p = ick2lib.Project('foo')
-        p.add_build_step('echo')
-        self.assertEqual(p.get_build_steps(), ['echo'])
-
-    def test_tracks_current_build_step(self):
-        p = ick2lib.Project('foo')
-        p.add_build_step('step1')
-        p.add_build_step('step2')
-        self.assertEqual(p.get_current_build_step('worker1'), None)
-        self.assertEqual(p.get_current_build_step('worker2'), None)
-        p.trigger_build()
-        self.assertEqual(p.get_current_build_step('worker1'), 'step1')
-        self.assertEqual(p.get_current_build_step('worker1'), 'step1')
-        self.assertEqual(p.get_current_build_step('worker2'), None)
-        p.finish_current_build_step()
-        self.assertEqual(p.get_current_build_step('worker1'), 'step2')
-        self.assertEqual(p.get_current_build_step('worker2'), None)
-        p.finish_current_build_step()
-        self.assertEqual(p.get_current_build_step('worker1'), None)
-        self.assertEqual(p.get_current_build_step('worker2'), None)
diff --git a/ick2lib/version.py b/ick2lib/version.py
deleted file mode 100644
index 20059f9..0000000
--- a/ick2lib/version.py
+++ /dev/null
@@ -1,2 +0,0 @@
-__version__ = "0.2.1+git"
-__version_info__ = (0, 2, 1, '+git')
diff --git a/run-debug b/run-debug
deleted file mode 100755
index a960e6a..0000000
--- a/run-debug
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-ick="$1"
-
-./controller \
-    --debug \
-    --pid-file=pid \
-    --port-file=port \
-    --log=log \
-    --projects="$ick" \
diff --git a/uwsgi.ini b/uwsgi.ini
deleted file mode 100644
index 2a427c4..0000000
--- a/uwsgi.ini
+++ /dev/null
@@ -1,12 +0,0 @@
-[uwsgi]
-logger = python
-http-socket = 0.0.0.0:12765
-master = true
-plugins = python
-wsgi-file = /usr/bin/ick2-controller
-pyargv=--projects /etc/ick2/projects.yaml --log /var/log/ick2/ick2.log
-uid = www-data
-gid = www-data
-workers = 1
-threads = 1
-buffer-size = 32768
diff --git a/yarns/000.yarn b/yarns/000.yarn
deleted file mode 100644
index 7a19824..0000000
--- a/yarns/000.yarn
+++ /dev/null
@@ -1,24 +0,0 @@
----
-title: Ick2 controller yarns
-...
-
-
-Introduction
-=============================================================================
-
-Ick2 will be a continuous integration system. Its core component is
-the **controller**, which does nothing, except decides what workers should
-do. It knows of the projects that can be built, and keeps track what
-step is being run on each project, and collects build output from the
-workers.
-
-This document specifies the yarn test scenarios for a minimal viable
-version of the controller: there will be a small number of project,
-each project builds a web site from source in git, using ikiwiki, and
-publishes the website on a server using rsync.
-
-The controller provides an HTTP API for controlling a build. The API
-is used by an external entity (such as the git server) to trigger a
-build, and by workers or worker proxies to request something to do,
-and to report results. The API may also eventually be used by end
-users to query status and results.
diff --git a/yarns/100-hello.yarn b/yarns/100-hello.yarn
deleted file mode 100644
index d719497..0000000
--- a/yarns/100-hello.yarn
+++ /dev/null
@@ -1,15 +0,0 @@
-# Controller smoke test
-
-This scenario is just for making sure we can, in our tests, start and
-stop the controller, and make requests to it.
-
-    SCENARIO controller smoke test
-    GIVEN a running controller instance
-
-    WHEN user calls GET /version
-    THEN response has status 200, and JSON body "{ "version": "1.0" }"
-
-    WHEN user calls GET /blatherskite
-    THEN response has status 404
-
-    FINALLY stop controller instance
diff --git a/yarns/200-build.yarn b/yarns/200-build.yarn
deleted file mode 100644
index 3465f50..0000000
--- a/yarns/200-build.yarn
+++ /dev/null
@@ -1,305 +0,0 @@
-Building a project
-=============================================================================
-
-One build, one worker
------------------------------------------------------------------------------
-
-This section uses the controller to walk through all the steps for a
-build. We start with some setup, defining a git repo and an ick
-project and starting the controller.
-
-    SCENARIO run a build with one worker
-
-    GIVEN a git repo foo.git with file index.mdwn containing
-    ... "hello, world\n"
-    AND a project foo, using foo.git, publishing to foo-web
-    AND a running controller instance
-
-Ensure controller knows of the project.
-
-    WHEN user calls GET /projects
-    THEN response has status 200, 
-    ... and JSON body "{ "projects": [ "foo" ] }"
-
-There is no job running now, so if the worker manager asks for work,
-it gets nothing.
-
-    WHEN worker manager calls GET /worker/bar
-    THEN response has status 200, and an empty body
-
-Trigger a new build. There is now work to do.
-
-    WHEN git server calls GET /projects/foo/+trigger
-    THEN response has status 200
-
-    WHEN worker manager calls GET /worker/bar
-    THEN response has status 200, and JSON body "{
-    ...     "project": "foo",
-    ...     "git": "foo.git",
-    ...     "shell": "ikiwiki --build"
-    ... }"
-
-Pretend a job is running, and send output to the controller. Don't send
-an exit code, since the pretend job hasn't finished. Check that the
-pretend output we sent ends up in the current build log.
-
-    WHEN worker manager calls POST /worker/bar/snippet,
-    ... with JSON body '{
-    ...     "project": "foo",
-    ...     "stdout": "ikiwiki build output",
-    ...     "stderr": "",
-    ...     "exit-code": null
-    ... }'
-    AND user calls GET /projects/foo/logs/current
-    THEN response has status 200, and text body "ikiwiki build output"
-
-The current build step hasn't changed.
-
-    WHEN worker manager calls GET /worker/bar
-    THEN response has status 200,
-    ... and JSON body "{
-    ...     "project": "foo",
-    ...     "git": "foo.git",
-    ...     "shell": "ikiwiki --build"
-    ... }"
-
-Pretend current command finishes. Make sure current log updates, and
-that we get a new thing to run.
-
-    WHEN worker manager calls POST /worker/bar/snippet,
-    ... with JSON body '{
-    ...     "project": "foo",
-    ...     "stdout": "|more output",
-    ...     "stderr": "",
-    ...     "exit-code": 0
-    ... }'
-    AND user calls GET /projects/foo/logs/current
-    THEN response has status 200, and an empty body
-
-    WHEN user calls GET /projects/foo/logs/previous
-    THEN response has status 200, 
-    ... and text body "ikiwiki build output|more output"
-
-    WHEN worker manager calls GET /worker/bar
-    THEN response has status 200,
-    ... and JSON body "{
-    ...     "project": "foo",
-    ...     "git": "foo.git",
-    ...     "shell": "rsync"
-    ... }"
-
-Tell worker the rsync command also finishes. After that, there should
-be nothing more to do. The current log should become empty, the
-previous log will contain the previously current log.
-
-    WHEN worker manager calls POST /worker/bar/snippet,
-    ... with JSON body '{
-    ...     "project": "foo",
-    ...     "stdout": "rsync output",
-    ...     "stderr": "",
-    ...     "exit-code": 0
-    ... }'
-
-    WHEN user calls GET /projects/foo/logs/current
-    THEN response has status 200, and an empty body
-
-    WHEN user calls GET /projects/foo/logs/previous
-    THEN response has status 200,  and text body "rsync output"
-
-    WHEN user calls GET /worker/bar
-    THEN response has status 200, and an empty body
-
-And we're done.
-
-    FINALLY stop controller instance
-
-
-Two builds, two workers
------------------------------------------------------------------------------
-
-This section runs two builds using two workers. The primary goal here
-is to make sure each worker gets consecutive steps for its own build,
-so that it runs all the steps in the same workspace.
-
-We start with some setup, defining a git repo and an ick project and
-starting the controller.
-
-    SCENARIO run two builds with two workers
-
-    GIVEN a git repo foo.git with file index.mdwn containing
-    ... "hello, world\n"
-    AND a git repo bar.git with file index.mdwn containing
-    ... "hello, bar\n"
-    AND a project foo, using foo.git, publishing to foo-web
-    AND a project bar, using bar.git, publishing to bar-web
-    AND a running controller instance
-
-Ensure controller knows of both projects. The list of project names
-should be sorted alphabetically.
-
-    WHEN user calls GET /projects
-    THEN response has status 200, 
-    ... and JSON body "{ "projects": [ "bar", "foo" ] }"
-
-There is no job running now, so if the worker manager asks for work,
-it gets nothing.
-
-    WHEN worker manager calls GET /worker/one
-    THEN response has status 200, and an empty body
-
-Trigger new builds on both projects. There is now work to do.
-
-    WHEN git server calls GET /projects/foo/+trigger
-    THEN response has status 200
-
-    WHEN worker manager calls GET /worker/one
-    THEN response has status 200, and JSON body "{
-    ...     "project": "foo",
-    ...     "git": "foo.git",
-    ...     "shell": "ikiwiki --build"
-    ... }"
-
-The second worker should still get nothing to do.
-
-    WHEN worker manager calls GET /worker/two
-    THEN response has status 200, and an empty body
-
-Trigger the other project, and the second worker gets something to do.
-
-    WHEN git server calls GET /projects/bar/+trigger
-    THEN response has status 200
-
-    WHEN worker manager calls GET /worker/two
-    THEN response has status 200, and JSON body "{
-    ...     "project": "bar",
-    ...     "git": "bar.git",
-    ...     "shell": "ikiwiki --build"
-    ... }"
-
-Pretend the build step for the first project is running, and send
-output to the controller. Don't send an exit code, since the pretend
-step hasn't finished. Check that the pretend output we sent ends up in
-the current build log.
-
-    WHEN worker manager calls POST /worker/one/snippet,
-    ... with JSON body '{
-    ...     "project": "foo",
-    ...     "stdout": "ikiwiki build output",
-    ...     "stderr": "",
-    ...     "exit-code": null
-    ... }'
-    AND user calls GET /projects/foo/logs/current
-    THEN response has status 200, and text body "ikiwiki build output"
-
-The current build step hasn't changed.
-
-    WHEN worker manager calls GET /worker/one
-    THEN response has status 200,
-    ... and JSON body "{
-    ...     "project": "foo",
-    ...     "git": "foo.git",
-    ...     "shell": "ikiwiki --build"
-    ... }"
-
-Pretend the build step finishes. Make sure current log updates, and
-that we get a new thing to run.
-
-    WHEN worker manager calls POST /worker/one/snippet,
-    ... with JSON body '{
-    ...     "project": "foo",
-    ...     "stdout": "|more output",
-    ...     "stderr": "",
-    ...     "exit-code": 0
-    ... }'
-    AND user calls GET /projects/foo/logs/current
-    THEN response has status 200, and an empty body
-
-    WHEN user calls GET /projects/foo/logs/previous
-    THEN response has status 200, 
-    ... and text body "ikiwiki build output|more output"
-
-    WHEN worker manager calls GET /worker/one
-    THEN response has status 200,
-    ... and JSON body "{
-    ...     "project": "foo",
-    ...     "git": "foo.git",
-    ...     "shell": "rsync"
-    ... }"
-
-The other worker is still running its step, and if it asks, it gets
-the same build step to run.
-
-    WHEN worker manager calls GET /worker/two
-    THEN response has status 200, and JSON body "{
-    ...     "project": "bar",
-    ...     "git": "bar.git",
-    ...     "shell": "ikiwiki --build"
-    ... }"
-
-Tell controller the rsync command of the first project also finishes.
-After that, there should be nothing more to do. The current log should
-become empty, the previous log will contain the previously current
-log.
-
-    WHEN worker manager calls POST /worker/one/snippet,
-    ... with JSON body '{
-    ...     "project": "foo",
-    ...     "stdout": "rsync output",
-    ...     "stderr": "",
-    ...     "exit-code": 0
-    ... }'
-
-    WHEN user calls GET /projects/foo/logs/current
-    THEN response has status 200, and an empty body
-
-    WHEN user calls GET /projects/foo/logs/previous
-    THEN response has status 200,  and text body "rsync output"
-
-    WHEN user calls GET /worker/one
-    THEN response has status 200, and an empty body
-
-Finish the other project build.
-
-    WHEN worker manager calls POST /worker/two/snippet,
-    ... with JSON body '{
-    ...     "project": "bar",
-    ...     "stdout": "ikiwiki output",
-    ...     "stderr": "",
-    ...     "exit-code": 0
-    ... }'
-    AND user calls GET /projects/bar/logs/current
-    THEN response has status 200, and an empty body
-
-    WHEN user calls GET /projects/bar/logs/previous
-    THEN response has status 200, 
-    ... and text body "ikiwiki output"
-
-    WHEN worker manager calls GET /worker/two
-    THEN response has status 200,
-    ... and JSON body "{
-    ...     "project": "bar",
-    ...     "git": "bar.git",
-    ...     "shell": "rsync"
-    ... }"
-
-    WHEN worker manager calls POST /worker/two/snippet,
-    ... with JSON body '{
-    ...     "project": "bar",
-    ...     "stdout": "second worker rsync output",
-    ...     "stderr": "",
-    ...     "exit-code": 0
-    ... }'
-
-    WHEN user calls GET /projects/bar/logs/current
-    THEN response has status 200, and an empty body
-
-    WHEN user calls GET /projects/bar/logs/previous
-    THEN response has status 200,  and text body 
-    ... "second worker rsync output"
-
-    WHEN user calls GET /worker/two
-    THEN response has status 200, and an empty body
-
-And we're done.
-
-    FINALLY stop controller instance
diff --git a/yarns/900.yarn b/yarns/900.yarn
deleted file mode 100644
index 8ee31f3..0000000
--- a/yarns/900.yarn
+++ /dev/null
@@ -1,96 +0,0 @@
-# Scenario step implementations
-
-# Manage controller instance
-
-    IMPLEMENTS GIVEN a running controller instance
-    controller = os.path.join(srcdir, 'controller')
-    cliapp.runcmd(
-        ['/usr/sbin/daemonize', '-c.',
-         controller, '--debug', '--pid-file=pid',
-         '--port-file=port', '--projects=ick.ick', '--log=log'])
-    vars['pid'] = cat('pid').strip()
-    vars['port'] = cat('port').strip()
-
-    IMPLEMENTS FINALLY stop controller instance
-    import signal
-    print 'killing process', repr(vars['pid'])
-    os.kill(int(vars['pid']), signal.SIGTERM)
-
-
-## Git repositories
-
-    IMPLEMENTS GIVEN a git repo (\S+) with file (\S+) containing "(.+)"
-    repo = yarnutils.get_next_match()
-    filename = yarnutils.get_next_match()
-    content = yarnutils.get_next_match()
-    pathname = os.path.join(repo, filename)
-    os.mkdir(repo)
-    git(repo, 'init', '.')
-    git(repo, 'config', 'user.email', 'user@example.com')
-    git(repo, 'config', 'user.name', 'J. Random User')
-    write(pathname, unescape(content))
-    git(repo, 'add', '.')
-    git(repo, 'commit', '-minitial')
-
-## Controller configuration
-
-    IMPLEMENTS GIVEN a project (\S+), using (\S+), publishing to (\S+)
-    project = yarnutils.get_next_match()
-    repo = yarnutils.get_next_match()
-
-    if os.path.exists('ick.ick'):
-        config = yaml.safe_load(open('ick.ick'))
-    else:
-        config = {
-            'projects': {}
-        }
-    config['projects'][project] = {
-        'git': repo,
-        'shell_steps': [
-            'ikiwiki --build',
-            'rsync',
-        ],
-    }
-    write('ick.ick', yaml.safe_dump(config))
-
-## API use
-
-    IMPLEMENTS WHEN (user|worker manager|git server) calls (\S+) (\S+)
-    who = yarnutils.get_next_match()
-    method = yarnutils.get_next_match()
-    path = yarnutils.get_next_match()
-    url = controller_url(vars['port'], path)
-    vars['status'], vars['body'] = request(method, url)
-
-    IMPLEMENTS WHEN worker manager calls (\S+) (\S+), with JSON body '(.+)'
-    method = yarnutils.get_next_match()
-    path = yarnutils.get_next_match()
-    body_text = yarnutils.get_next_match()
-    url = controller_url(vars['port'], path)
-    body = parse_json(body_text)
-    vars['status'], vars['body'] = request(method, url, body=body_text)
-
-    IMPLEMENTS THEN response has status (\d+), and an empty body
-    status = yarnutils.get_next_match()
-    yarnutils.assertEqual(int(status), int(vars['status']))
-    yarnutils.assertEqual(vars['body'], '')
-
-    IMPLEMENTS THEN response has status (\d+), and text body "(.+)"
-    status = yarnutils.get_next_match()
-    bodypat = yarnutils.get_next_match()
-    yarnutils.assertEqual(int(status), int(vars['status']))
-    yarnutils.assertEqual(vars['body'], unescape(bodypat) )
-
-    IMPLEMENTS THEN response has status (\d+), and JSON body "(.+)"
-    status = yarnutils.get_next_match()
-    bodytext = yarnutils.get_next_match()
-    print 'varsbody:', repr(vars['body'])
-    print 'bodytext:', repr(bodytext)
-    bodyjson = parse_json(bodytext)
-    print 'bodyjson:', repr(bodyjson)
-    yarnutils.assertEqual(int(status), int(vars['status']))
-    yarnutils.assertEqual(parse_json(vars['body']), parse_json(bodytext) )
-
-    IMPLEMENTS THEN response has status (\d+)
-    status = yarnutils.get_next_match()
-    yarnutils.assertEqual(int(status), int(vars['status']))
diff --git a/yarns/lib.py b/yarns/lib.py
deleted file mode 100644
index fc4acfc..0000000
--- a/yarns/lib.py
+++ /dev/null
@@ -1,111 +0,0 @@
-import errno
-import json
-import os
-import StringIO
-import time
-
-import cliapp
-import requests
-import yaml
-import yarnutils
-
-
-datadir = os.environ['DATADIR']
-srcdir = os.environ['SRCDIR']
-
-vars = yarnutils.Variables(datadir)
-
-
-MAX_CAT_TIME = 5 # seconds
-def cat(filename):
-    start = time.time()
-    while time.time() - start < MAX_CAT_TIME:
-        try:
-            with open(filename) as f:
-                data = f.read()
-                if len(data) == 0:
-                    continue
-                return data
-        except (IOError, OSError) as e:
-            if e.errno == errno.ENOENT:
-                continue
-            raise
-    raise Exception("cat took more then %s seconds" % MAX_CAT_TIME)
-
-
-def write(filename, content):
-    with open(filename, 'w') as f:
-        f.write(content)
-
-
-def git(repo, *argv):
-    return cliapp.runcmd(['git'] + list(argv), cwd=repo)
-
-
-def controller_url(port, path):
-    return 'http://localhost:{}{}'.format(port, path)
-
-
-def request(method, url, body=None):
-    funcs = {
-        'POST': requests.post,
-        'PUT': requests.put,
-        'GET': requests.get,
-        'DELETE': requests.delete,
-    }
-
-    headers = {
-        'Content-Type': 'application/json',
-    }
-
-    response = funcs[method](
-        url,
-        headers=headers,
-        data=body,
-    )
-
-    return response.status_code, response.text
-
-
-def parse_json(text):
-    return json.loads(text, object_pairs_hook=dictify)
-
-
-
-def dictify(pairs):
-    return {
-        stringify(key): stringify(value)
-        for key, value in pairs
-    }
-
-
-def stringify(x):
-    if isinstance(x, unicode):
-        return str(x)
-    if isinstance(x, list):
-        return [stringify(y) for y in x]
-    if isinstance(x, dict):
-        return {
-            stringify(key): stringify(value)
-            for key, value in pairs
-        }
-    return x
-
-
-def parse_yaml(text):
-    f = StringIO.StringIO(text)
-    return yaml.safe_load(stream=f)
-
-
-def unescape(text):
-    def helper(text):
-        while text:
-            if text.startswith('\\n'):
-                skip = 2
-                answer = '\n'
-            else:
-                skip = 1
-                answer = text[0]
-            text = text[skip:]
-            yield answer
-    return ''.join(helper(text))