Merge branch 'liw/create_workspace_step'

13 files changed, 494 insertions, 113 deletions

diff --git a/NEWS b/NEWS
index 4a5f772..83507bb 100644
--- a/NEWS
+++ b/NEWS
@@ -20,13 +20,25 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 Version 0.21+git, not yet released
 ----------------------------------
 
-* Add a `where: chroot` field to pipeline actions to cause the
-  shell/python snippet to run in a chroot using the workspace as the
-  root directory.
-
-* A pipeline can now invoke the `archive` action to create a
-  compressed tarball of the contents of the workspace, and store it in
-  the blob service.
+* The worker manager can now run things in a chroot, using the
+  workspace as the root directory. Add a `where: chroot` field to
+  the `shell` or `python` pipeline actions.
+
+* The worker manager can now archive the contents of the workspace and
+  store it in the blob service. It can also retrieve a blob an unpack
+  it into `/var/lib/ick/systree`, and run builds using that directory
+  as the root of a container.
+
+* The worker manager is now significantly more efficient when
+  reporting build output to the controller. Previously, this would be
+  done every time the stdout or stderr of the build command produced
+  any output. In practice this meant an HTTP request every few bytes
+  of output. Output is now buffered (up to a kibibyte per output
+  stream), reducing the overhead rather a lot.
+
+* `icktool` has a new subcommand, `make-it-so`, which reads a YAML
+  file which lists all projects and pipelines, and creates or updates
+  them via the controller API.
 
 Version 0.21, released 2017-12-27
 ----------------------------------
diff --git a/debian/changelog b/debian/changelog
index 07fe9db..8733bea 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 ick2 (0.21+git-1) UNRELEASED; urgency=medium
 
   * New upstream version.
+  * Create /var/lib/ick2/systree.
 
  -- Lars Wirzenius <liw@liw.fi>  Wed, 27 Dec 2017 18:35:28 +0200
 
diff --git a/debian/ick2.postinst b/debian/ick2.postinst
index 3e70087..822006b 100644
--- a/debian/ick2.postinst
+++ b/debian/ick2.postinst
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2017 Lars Wirzenius
+# Copyright 2017-2018 Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -47,6 +47,9 @@ install -d  -m 0755 -o _ick -g _ick /var/lib/ick/state
 # Create worker-manager workspace
 install -d  -m 0755 -o _ickwm -g _ickwm /var/lib/ick/workspace
 
+# Create worker-manager systree
+install -d  -m 0755 -o _ickwm -g _ickwm /var/lib/ick/systree
+
 # Create blob service storage
 install -d  -m 0755 -o _ickbs -g _ickbs /var/lib/ick/blobs
 
diff --git a/ick2/__init__.py b/ick2/__init__.py
index 9af0521..f5770f6 100644
--- a/ick2/__init__.py
+++ b/ick2/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -22,6 +22,7 @@ from .state import (
     Workers,
     Projects,
     PipelineInstances,
+    Builds,
 )
 from .exceptions import (
     BadUpdate,
diff --git a/ick2/state.py b/ick2/state.py
index 806b2a7..2456bfc 100644
--- a/ick2/state.py
+++ b/ick2/state.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -40,9 +40,9 @@ class ControllerState:
     def get_resource_directory(self, type_name):
         return os.path.join(self.get_state_directory(), type_name)
 
-    def get_resource_filename(self, type_name, project_name):
-        return os.path.join(
-            self.get_resource_directory(type_name), project_name + '.yaml')
+    def get_resource_filename(self, type_name, resource_name):
+        dirname = self.get_resource_directory(type_name)
+        return os.path.join(dirname, resource_name + '.yaml')
 
     def load_resources(self, type_name):
         assert self._statedir is not None
@@ -107,10 +107,14 @@ class ResourceStore:  # pragma: no cover
         self._category = category
         self._name_field = name_field
 
+    def get_resource_name(self, name):
+        return str(name)
+
     def list(self):
         return self._state.get_resources(self._category)
 
     def get(self, name):
+        name = self.get_resource_name(name)
         try:
             return self._state.get_resource(self._category, name)
         except ick2.NotFound:
@@ -119,10 +123,11 @@ class ResourceStore:  # pragma: no cover
             }
 
     def add(self, name, resource):
+        name = self.get_resource_name(name)
         self._state.add_resource(self._category, name, resource)
 
     def update(self, resource):
-        name = resource[self._name_field]
+        name = self.get_resource_name(resource[self._name_field])
         try:
             self._state.get_resource(self._category, name)
         except ick2.NotFound:
@@ -155,6 +160,18 @@ class Projects(ResourceStore):  # pragma: no cover
         self.update(project)
 
 
+class Builds(ResourceStore):  # pragma: no cover
+
+    def __init__(self, state):
+        super().__init__(state, 'builds', 'build_id')
+
+    def get_builds(self):
+        return self.list()
+
+    def update_build(self, build):
+        self.update(build)
+
+
 class PipelineInstances(ResourceStore):  # pragma: no cover
 
     def __init__(self, state):
diff --git a/ick2/workapi.py b/ick2/workapi.py
index 9c692e8..8237ff4 100644
--- a/ick2/workapi.py
+++ b/ick2/workapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -23,6 +23,7 @@ class WorkAPI(ick2.APIbase):
         self._workers = ick2.Workers(state)
         self._projects = ick2.Projects(state)
         self._pinstances = ick2.PipelineInstances(state)
+        self._builds = ick2.Builds(state)
         self._type_name = 'work'
 
     def get_routes(self, path):  # pragma: no cover
@@ -55,17 +56,17 @@ class WorkAPI(ick2.APIbase):
 
                 self._start_build(project, pipeline, worker, build_id)
                 self._start_log(build_id)
+                build = self._get_build(build_id)
+                actions = build['actions']
+                current_action = build['current_action']
 
-                index = 0
                 doing = {
                     'build_id': build_id,
                     'worker': worker,
                     'project': project['project'],
                     'pipeline': pipeline['name'],
                     'parameters': project.get('parameters', {}),
-                    'fresh_workspace': True,
-                    'step': pipeline['actions'][index],
-                    'step_index': index,
+                    'step': actions[current_action],
                     'log': '/logs/{}'.format(build_id),
                 }
 
@@ -100,6 +101,10 @@ class WorkAPI(ick2.APIbase):
     def _start_build(self, project, pipeline, worker, build_id):
         ick2.log.log('info', msg_text='Starting new build', build_id=build_id)
         parameters = project.get('parameters', {})
+        create_workspace = {
+            'action': 'create_workspace',
+        }
+        actions = [create_workspace] + list(pipeline['actions'])
         build = {
             'build_id': build_id,
             'log': '/logs/{}'.format(build_id),
@@ -108,10 +113,15 @@ class WorkAPI(ick2.APIbase):
             'pipeline': pipeline['name'],
             'parameters': parameters,
             'status': 'building',
+            'actions': actions,
+            'current_action': 0,
         }
-        self._state.add_resource('builds', str(build_id), build)
+        self._builds.add(build_id, build)
         return build_id
 
+    def _get_build(self, build_id):
+        return self._builds.get(build_id)
+
     def _start_log(self, build_id):
         ick2.log.log('info', msg_text='Starting new log', build_id=build_id)
         log = {
@@ -132,27 +142,22 @@ class WorkAPI(ick2.APIbase):
             update['project'], update['pipeline'])
         self._append_to_build_log(update)
 
-        ick2.log.log(
-            'trace',
-            msg_text='xxx update_work',
-            update=update,
-            project=project,
-            pipeline=pipeline,
-            doing=doing)
-
         exit_code = update.get('exit_code')
         if exit_code == 0:
-            index = doing['step_index'] + 1
-            actions = pipeline['actions']
-            if index >= len(actions):
+            build_id = doing['build_id']
+            build = self._get_build(build_id)
+            actions = build['actions']
+            current_action = build['current_action']
+            if current_action + 1 >= len(actions):
                 pipeline['status'] = 'idle'
                 self._update_pipeline(project, pipeline)
                 doing = {}
                 self._finish_build(update)
             else:
-                doing['step_index'] = index
-                doing['step'] = dict(actions[index])
-                doing['fresh_workspace'] = False
+                index = current_action + 1
+                build['current_action'] = index
+                self._update_build(build)
+                doing['step'] = actions[index]
 
             worker_state = {
                 'worker': update['worker'],
@@ -205,10 +210,15 @@ class WorkAPI(ick2.APIbase):
                 log['log'] += text
         self._state.update_resource('log', str(build_id), log)
 
+    def _update_build(self, build):
+        self._builds.update_build(build)
+
     def _finish_build(self, update):
-        build = self._state.get_resource('builds', str(update['build_id']))
+        build_id = update['build_id']
+        build = self._get_build(build_id)
         build['status'] = update['exit_code']
-        self._state.update_resource('builds', str(update['build_id']), build)
+        build['current_action'] = None
+        self._update_build(build)
 
     def create(self, body, **kwargs):  # pragma: no cover
         pass
diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py
index d7ebfa9..48c9bbb 100644
--- a/ick2/workapi_tests.py
+++ b/ick2/workapi_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
@@ -86,11 +86,9 @@ class WorkAPITests(unittest.TestCase):
             'parameters': {
                 'foo': 'bar',
             },
-            'fresh_workspace': True,
             'step': {
-                'shell': 'step-1',
+                'action': 'create_workspace',
             },
-            'step_index': 0,
             'log': '/logs/1',
         }
         self.assertEqual(work.get_work('asterix'), expected)
@@ -113,11 +111,9 @@ class WorkAPITests(unittest.TestCase):
             'parameters': {
                 'foo': 'bar',
             },
-            'fresh_workspace': True,
             'step': {
-                'shell': 'step-1',
+                'action': 'create_workspace',
             },
-            'step_index': 0,
             'log': '/logs/1',
         }
         self.assertEqual(work.get_work('asterix'), expected)
@@ -144,9 +140,15 @@ class WorkAPITests(unittest.TestCase):
         work.update_work(done)
 
         # We should get the next step now.
+        expected['step'] = {'shell': 'step-1'}
+        self.assertEqual(work.get_work('asterix'), expected)
+
+        # Finish the step.
+        done['exit_code'] = 0
+        work.update_work(done)
+
+        # We should get the next step now.
         expected['step'] = {'shell': 'step-2'}
-        expected['step_index'] = 1
-        expected['fresh_workspace'] = False
         self.assertEqual(work.get_work('asterix'), expected)
 
         # Finish the step.
@@ -176,11 +178,9 @@ class WorkAPITests(unittest.TestCase):
             'parameters': {
                 'foo': 'bar',
             },
-            'fresh_workspace': True,
             'step': {
-                'shell': 'step-1',
+                'action': 'create_workspace',
             },
-            'step_index': 0,
             'log': '/logs/1',
         }
         self.assertEqual(work.get_work('asterix'), expected)
diff --git a/icktool b/icktool
index bb76aaa..35cb251 100755
--- a/icktool
+++ b/icktool
@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-# Copyright 2017  Lars Wirzenius
+# Copyright 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -163,6 +163,22 @@ class Icktool(cliapp.Application):
                 latest = build
         return latest
 
+    def cmd_make_it_so(self, args):
+        obj = self._read_object()
+
+        projects = self._new_rc('/projects', 'project')
+        self._make_it_so(projects, obj.get('projects', []))
+
+        pipelines = self._new_rc('/pipelines', 'name')
+        self._make_it_so(pipelines, obj.get('pipelines', []))
+
+    def _make_it_so(self, rc, objs):
+        for obj in objs:
+            if rc.exists(obj):
+                rc.update(obj)
+            else:
+                rc.create(obj)
+
     def cmd_list_projects(self, args):
         self._prettyson(self._get_projects())
 
@@ -506,6 +522,10 @@ class ResourceCommands:
         self._report(code, 200, text)
         return json.loads(text)
 
+    def exists(self, obj):
+        code, text = self._api.get(self._id_path(obj[self._name]))
+        return code == 200
+
     def delete(self, name):
         code, text = self._api.delete(self._id_path(name))
         self._report(code, 200, text)
diff --git a/worker_manager b/worker_manager
index d29b597..1cc012e 100755
--- a/worker_manager
+++ b/worker_manager
@@ -1,5 +1,5 @@
 #!/usr/bin/python3
-# Copyright 2017  Lars Wirzenius
+# Copyright 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -80,6 +80,13 @@ class WorkerManager(cliapp.Application):
             default='/var/lib/ick/workspace',
         )
 
+        self.settings.string(
+            ['systree'],
+            'use DIR as the system tree for containers',
+            metavar='DIR',
+            default='/var/lib/ick/systree',
+        )
+
     def process_args(self, args):
         self.settings.require('name')
         self.settings.require('controller')
@@ -88,8 +95,9 @@ class WorkerManager(cliapp.Application):
         url = self.settings['controller']
         tg = TokenGenerator()
         tg.set_key(self.settings['token-key'])
-        api = ContainerAPI(name, url, tg)
-        worker = Worker(name, api, self.settings['workspace'])
+        api = ControllerAPI(name, url, tg)
+        worker = Worker(
+            name, api, self.settings['workspace'], self.settings['systree'])
 
         logging.info('Worker manager %s starts, controller is %s', name, url)
 
@@ -106,7 +114,7 @@ class WorkerManager(cliapp.Application):
         time.sleep(secs)
 
 
-class ContainerAPI:
+class ControllerAPI:
 
     def __init__(self, name, url, token_generator):
         self._name = name
@@ -146,6 +154,12 @@ class ContainerAPI:
         headers = self.get_auth_headers()
         code = self._httpapi.put(url, headers, blob)
 
+    def download_blob(self, blob_id):
+        logging.info('Download blob %s', blob_id)
+        url = self.url('/blobs/{}'.format(blob_id))
+        headers = self.get_auth_headers()
+        return self._httpapi.get_blob(url, headers)
+
     def url(self, path):
         return '{}{}'.format(self._url, path)
 
@@ -179,6 +193,12 @@ class HttpApi:
             return None
         return r.json()
 
+    def get_blob(self, url, headers):
+        r = self._session.get(url, headers=headers, verify=False)
+        if not r.ok:
+            return None
+        return r.content
+
 
 class TokenGenerator:
 
@@ -242,10 +262,11 @@ class TokenGenerator:
 
 class Worker:
 
-    def __init__(self, name, api, workspace):
+    def __init__(self, name, api, workspace, systree):
         self._name = name
         self._api = api
         self._workspace = workspace
+        self._systree = systree
 
     def do_work(self, work):
 
@@ -257,16 +278,12 @@ class Worker:
         step = work['step']
         logging.info('Running step: %r', step)
         exit_code = 0
-        if work.get('fresh_workspace'):
-            logging.info('Make an empty workspace')
-            cleaner = WorkspaceCleaner(None, self._workspace, post)
-            exit_code = cleaner.do(work)
         if exit_code == 0:
             klass = self.worker_factory(step)
             if klass is None:
                 exit_code = -1
             else:
-                worker = klass(self._api, self._workspace, post)
+                worker = klass(self._api, self._workspace, self._systree, post)
                 exit_code = worker.do(work)
         self.finish_work(work, exit_code)
 
@@ -286,14 +303,19 @@ class Worker:
         return time.strftime('%Y-%m-%dT%H:%M:%S')
 
     def worker_factory(self, step):
-        if 'shell' in step:
-            return ShellWorker
-        elif 'python' in step:
-            return PythonWorker
-        elif 'debootstrap' in step:
-            return DebootstrapWorker
-        elif 'archive' in step:
-            return WorkspaceArchiver
+        table = [
+            ('shell', None, ShellWorker),
+            ('python', None, PythonWorker),
+            ('debootstrap', None, DebootstrapWorker),
+            ('archive', None, WorkspaceArchiver),
+            ('action', 'populate_systree', SystreePopulator),
+            ('action', 'create_workspace', WorkspaceCleaner),
+        ]
+
+        for key, value, klass in table:
+            if key in step and (value is None or step[key] == value):
+                return klass
+
         logging.warning('Cannot find worker for %s', step)
         return None
 
@@ -303,11 +325,54 @@ class Worker:
         self._api.report_work(s)
 
 
+class Runner:
+
+    def __init__(self, post_output):
+        self._post = post_output
+        self._buffers = {'stdout': '', 'stderr': ''}
+        self._maxbuf = 2**10
+        self._timeout = 1.0
+
+    def runcmd(self, argv, **kwargs):
+        logging.debug('Runner.runcmd: argv=%r %r', argv, kwargs)
+        exit_code, _, _ = cliapp.runcmd_unchecked(
+            argv,
+            stdout_callback=self.capture_stdout,
+            stderr_callback=self.capture_stderr,
+            timeout_callback=self.flush,
+            **kwargs
+        )
+        self.flush()
+        logging.debug('Runner.runcmd: finished, exit_code=%r', exit_code)
+        return exit_code
+
+    def capture_stdout(self, data):
+        return self.capture('stdout', data)
+
+    def capture_stderr(self, data):
+        return self.capture('stderr', data)
+
+    def capture(self, stream_name, data):
+        self._buffers[stream_name] += data.decode('UTF-8')
+        if len(self._buffers[stream_name]) >= self._maxbuf:
+            self.flush()
+        return None
+
+    def flush(self):
+        for stream in self._buffers:
+            buf = self._buffers[stream]
+            self._buffers[stream] = ''
+            if buf:
+                logging.debug('Posting %d bytes of %s', len(buf), stream)
+                return self._post(stream, buf)
+
+
 class WorkerBase:
 
-    def __init__(self, api, workspace, post):
+    def __init__(self, api, workspace, systree, post):
         self._api = api
         self._workspace = workspace
+        self._systree = systree
         self._post = post
 
     def do(self, work):
@@ -317,16 +382,16 @@ class WorkerBase:
         if self.where(work) == 'chroot':
             logging.debug('CHROOT REQUESTED')
             argv = ['sudo', 'chroot', self._workspace] + argv
+        elif self.where(work) == 'container':
+            logging.debug('CONTAINER REQUESTED')
+            argv = [
+                'sudo', 'systemd-nspawn', '-D', self._systree,
+                '--bind', self._workspace,
+            ] + argv
         else:
-            logging.debug('NOT IN CHROOT')
-        logging.debug('running: %r', argv)
-        exit_code, _, _ = cliapp.runcmd_unchecked(
-            argv,
-            stdout_callback=self.report_stdout,
-            stderr_callback=self.report_stderr,
-            cwd=self._workspace,
-        )
-        return exit_code
+            logging.debug('HOST REQUESTED')
+        runner = Runner(self._post)
+        return runner.runcmd(argv, cwd=self._workspace)
 
     def report_stdout(self, data):
         return self._post('stdout', data.decode('UTF-8'))
@@ -351,7 +416,9 @@ class WorkerBase:
 class ShellWorker(WorkerBase):
 
     def get_argv(self, work, params_text):
-        code_snippet = work['step']['shell']
+        step = work['step']
+        code_snippet = step['shell']
+        where = step.get('where', 'host')
         prefix = 'params() { echo -n "%s" | base64 -d; }\n' % params_text
         return ['bash', '-exuc', prefix + code_snippet]
 
@@ -428,5 +495,49 @@ class WorkspaceArchiver(WorkerBase):
         assert False
 
 
+class SystreePopulator(WorkerBase):
+
+    systree_dir = '/var/lib/ick/systree'
+
+    def do(self, work):
+        step = work['step']
+        systree_name = step.get('systree_name')
+        if not systree_name:
+            self.report(
+                b'No systree_name field in action, no systree population\n')
+            return 1
+
+        self.make_directory_empty(self.systree_dir)
+        tarball = self._api.download_blob(systree_name)
+        self.unpack_systree(tarball, self.systree_dir)
+
+        self.report(b'Systree has been populated\n')
+        return 0
+
+    def make_directory_empty(self, dirname):
+        return self.execute_argv(['sudo', 'find', '-delete'], cwd=dirname)
+
+    def unpack_systree(self, tarball, dirname):
+        return self.execute_argv(
+            ['sudo', 'tar', '-zxf', '-', '-C', dirname],
+            feed_stdin=tarball,
+        )
+
+    def execute_argv(self, argv, **kwargs):
+        exit_code, _, _ = cliapp.runcmd_unchecked(
+            argv,
+            stdout_callback=self.report,
+            stderr_callback=self.report,
+            **kwargs,
+        )
+        return exit_code
+
+    def report(self, data):
+        self._post('stdout', data.decode('UTF-8'))
+
+    def get_argv(self, work, params_text):
+        assert False
+
+
 if __name__ == '__main__':
     WorkerManager(version=ick2.__version__).run()
diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn
index c8746b4..b18a0e6 100644
--- a/yarns/400-build.yarn
+++ b/yarns/400-build.yarn
@@ -1,6 +1,6 @@
 <!--
 
-Copyright 2017 Lars Wirzenius
+Copyright 2017-2018 Lars Wirzenius
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -108,6 +108,9 @@ Worker wants work and gets the first step to run. If the worker asks
 again, it gets the same answer. **FIXME: should the name of the worker
 be in the path or can we get it in the access token?**
 
+Note that the controller has inserted a special additional step to get
+the worker to construct a new workspace for the build.
+
     WHEN worker-manager makes request GET /work/obelix
     THEN result has status code 200
     AND body matches
@@ -120,11 +123,9 @@ be in the path or can we get it in the access token?**
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
-    ...     "fresh_workspace": true,
     ...     "step": {
-    ...         "shell": "day 1"
-    ...     },
-    ...     "step_index": 0
+    ...         "action": "create_workspace"
+    ...     }
     ... }
 
     WHEN worker-manager makes request GET /work/obelix
@@ -139,11 +140,9 @@ be in the path or can we get it in the access token?**
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
-    ...     "fresh_workspace": true,
     ...     "step": {
-    ...         "shell": "day 1"
-    ...     },
-    ...     "step_index": 0
+    ...         "action": "create_workspace"
+    ...     }
     ... }
 
 User can now see pipeline is running and which worker is building it.
@@ -169,10 +168,8 @@ User can now see pipeline is running and which worker is building it.
     ...         "parameters": {
     ...             "foo": "bar"
     ...         },
-    ...         "fresh_workspace": true,
-    ...         "step_index": 0,
     ...         "step": {
-    ...             "shell": "day 1"
+    ...             "action": "create_workspace"
     ...         }
     ...     }
     ... }
@@ -188,6 +185,12 @@ User can now see pipeline is running and which worker is building it.
     ...             "worker": "obelix",
     ...             "project": "rome",
     ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": 0,
     ...             "parameters": {
     ...                 "foo": "bar"
     ...             },
@@ -202,7 +205,42 @@ User can now see pipeline is running and which worker is building it.
     AND result has header Content-Type: text/plain
     AND body text is ""
 
-Worker reports some build output. Note the null exit code.
+Worker reports workspace creation is done. Note the zero exit code.
+
+    WHEN worker-manager makes request POST /work with a valid token and body
+    ... {
+    ...     "build_id": 1,
+    ...     "worker": "obelix",
+    ...     "project": "rome",
+    ...     "pipeline": "construct",
+    ...     "exit_code": 0,
+    ...     "stdout": "",
+    ...     "stderr": "",
+    ...     "timestamp": "2017-10-27T17:08:49"
+    ... }
+    THEN result has status code 201
+
+Worker requests more work, and gets the first actual build step.
+
+    WHEN worker-manager makes request GET /work/obelix
+    THEN result has status code 200
+    AND body matches
+    ... {
+    ...     "build_id": 1,
+    ...     "log": "/logs/1",
+    ...     "worker": "obelix",
+    ...     "project": "rome",
+    ...     "pipeline": "construct",
+    ...     "parameters": {
+    ...         "foo": "bar"
+    ...     },
+    ...     "step": {
+    ...         "shell": "day 1"
+    ...     }
+    ... }
+
+Worker reports some build output. Note the null exit code. The step
+hasn't finished yet.
 
     WHEN worker-manager makes request POST /work with a valid token and body
     ... {
@@ -217,7 +255,8 @@ Worker reports some build output. Note the null exit code.
     ... }
     THEN result has status code 201
 
-Still the same job, since the first build step didnt't finish.
+Worker-manager still gets the same step, since the first build step
+didnt't finish.
 
     WHEN worker-manager makes request GET /work/obelix
     THEN result has status code 200
@@ -231,11 +270,9 @@ Still the same job, since the first build step didnt't finish.
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
-    ...     "fresh_workspace": true,
     ...     "step": {
     ...         "shell": "day 1"
-    ...     },
-    ...     "step_index": 0
+    ...     }
     ... }
 
 The build log is immediately accessible.
@@ -265,6 +302,34 @@ Report the step is done, and successfully.
     AND result has header Content-Type: text/plain
     AND body text is "hey ho, hey ho\n"
 
+The build status now shows the next step as the active one.
+
+    WHEN user makes request GET /builds
+    THEN result has status code 200
+    AND body matches
+    ... {
+    ...     "builds": [
+    ...         {
+    ...             "build_id": 1,
+    ...             "log": "/logs/1",
+    ...             "worker": "obelix",
+    ...             "project": "rome",
+    ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": 2,
+    ...             "parameters": {
+    ...                 "foo": "bar"
+    ...             },
+    ...             "status": "building",
+    ...             "log": "/logs/1"
+    ...         }
+    ...     ]
+    ... }
+
 Now there's another step to do.
 
     WHEN worker-manager makes request GET /work/obelix
@@ -279,11 +344,9 @@ Now there's another step to do.
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
-    ...     "fresh_workspace": false,
     ...     "step": {
     ...         "shell": "day 2"
-    ...     },
-    ...     "step_index": 1
+    ...     }
     ... }
 
 User sees changed status.
@@ -301,8 +364,6 @@ User sees changed status.
     ...         "parameters": {
     ...             "foo": "bar"
     ...         },
-    ...         "fresh_workspace": false,
-    ...         "step_index": 1,
     ...         "step": {
     ...             "shell": "day 2"
     ...         },
@@ -337,7 +398,8 @@ The pipeline status indicates success.
     THEN result has status code 200
     AND body matches { "status": "idle" }
 
-Also, there's a build with a log.
+Also, there's a build with a log. Also, the build status shows there's
+no current action.
 
     WHEN user makes request GET /builds
     THEN result has status code 200
@@ -350,6 +412,12 @@ Also, there's a build with a log.
     ...             "worker": "obelix",
     ...             "project": "rome",
     ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": null,
     ...             "parameters": {
     ...                 "foo": "bar"
     ...             },
@@ -368,6 +436,12 @@ Also, there's a build with a log.
     ...     "worker": "obelix",
     ...     "project": "rome",
     ...     "pipeline": "construct",
+    ...     "actions": [
+    ...         { "action": "create_workspace" },
+    ...         { "shell": "day 1" },
+    ...         { "shell": "day 2" }
+    ...     ],
+    ...     "current_action": null,
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
@@ -398,11 +472,9 @@ Start build again. This should become build number 2.
     ...     "parameters": {
     ...         "foo": "bar"
     ...     },
-    ...     "fresh_workspace": true,
     ...     "step": {
-    ...         "shell": "day 1"
-    ...     },
-    ...     "step_index": 0
+    ...         "action": "create_workspace"
+    ...     }
     ... }
 
     WHEN user makes request GET /builds
@@ -416,6 +488,12 @@ Start build again. This should become build number 2.
     ...             "worker": "obelix",
     ...             "project": "rome",
     ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": null,
     ...             "parameters": {
     ...                 "foo": "bar"
     ...             },
@@ -427,6 +505,12 @@ Start build again. This should become build number 2.
     ...             "worker": "obelix",
     ...             "project": "rome",
     ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": 0,
     ...             "parameters": {
     ...                 "foo": "bar"
     ...             },
@@ -442,6 +526,36 @@ Start build again. This should become build number 2.
     ...     "project": "rome",
     ...     "pipeline": "construct",
     ...     "exit_code": 0,
+    ...     "stdout": "",
+    ...     "stderr": "",
+    ...     "timestamp": "2017-10-27T17:08:49"
+    ... }
+    THEN result has status code 201
+
+    WHEN worker-manager makes request GET /work/obelix
+    THEN result has status code 200
+    AND body matches
+    ... {
+    ...     "build_id": 2,
+    ...     "log": "/logs/2",
+    ...     "worker": "obelix",
+    ...     "project": "rome",
+    ...     "pipeline": "construct",
+    ...     "parameters": {
+    ...         "foo": "bar"
+    ...     },
+    ...     "step": {
+    ...         "shell": "day 1"
+    ...     }
+    ... }
+
+    WHEN worker-manager makes request POST /work with a valid token and body
+    ... {
+    ...     "build_id": 2,
+    ...     "worker": "obelix",
+    ...     "project": "rome",
+    ...     "pipeline": "construct",
+    ...     "exit_code": 0,
     ...     "stdout": "hey ho",
     ...     "stderr": "",
     ...     "timestamp": "2017-10-27T17:08:49"
@@ -475,6 +589,12 @@ Start build again. This should become build number 2.
     ...             "worker": "obelix",
     ...             "project": "rome",
     ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": null,
     ...             "parameters": {
     ...                 "foo": "bar"
     ...             },
@@ -486,6 +606,12 @@ Start build again. This should become build number 2.
     ...             "worker": "obelix",
     ...             "project": "rome",
     ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": null,
     ...             "parameters": {
     ...                 "foo": "bar"
     ...             },
diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn
index bc0a5ab..433c3e0 100644
--- a/yarns/500-build-fail.yarn
+++ b/yarns/500-build-fail.yarn
@@ -1,6 +1,6 @@
 <!--
 
-Copyright 2017 Lars Wirzenius
+Copyright 2017-2018 Lars Wirzenius
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -88,11 +88,9 @@ Worker wants work and gets the first step to run.
     ...     "project": "rome",
     ...     "pipeline": "construct",
     ...     "parameters": {},
-    ...     "fresh_workspace": true,
     ...     "step": {
-    ...         "shell": "day 1"
-    ...     },
-    ...     "step_index": 0
+    ...         "action": "create_workspace"
+    ...     }
     ... }
 
 Worker reports some build output. Note the exit code indicating
@@ -147,6 +145,12 @@ Also, there's a build with a log.
     ...             "worker": "obelix",
     ...             "project": "rome",
     ...             "pipeline": "construct",
+    ...             "actions": [
+    ...                 { "action": "create_workspace" },
+    ...                 { "shell": "day 1" },
+    ...                 { "shell": "day 2" }
+    ...             ],
+    ...             "current_action": null,
     ...             "parameters": {},
     ...             "status": 1,
     ...             "log": "/logs/1"
@@ -163,6 +167,12 @@ Also, there's a build with a log.
     ...     "worker": "obelix",
     ...     "project": "rome",
     ...     "pipeline": "construct",
+    ...     "actions": [
+    ...         { "action": "create_workspace" },
+    ...         { "shell": "day 1" },
+    ...         { "shell": "day 2" }
+    ...     ],
+    ...     "current_action": null,
     ...     "parameters": {},
     ...     "status": 1,
     ...     "log": "/logs/1"
diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn
index 1a42198..39b590b 100644
--- a/yarns/900-implements.yarn
+++ b/yarns/900-implements.yarn
@@ -1,6 +1,6 @@
 <!--
 
-Copyright 2017 Lars Wirzenius
+Copyright 2017-2018 Lars Wirzenius
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -108,7 +108,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     expected_text = get_next_match()
     expected = json.loads(expected_text)
     actual = json.loads(vars['body'])
-    assertEqual(expected, actual)
+    print('expected', json.dumps(expected, indent=4))
+    print('actual', json.dumps(actual, indent=4))
+    diff = dict_diff(expected, actual)
+    if diff is not None:
+        print(diff)
+        assert 0
 
     IMPLEMENTS THEN body text is "(.*)"
     expected = unescape(get_next_match())
diff --git a/yarns/lib.py b/yarns/lib.py
index b19ccf3..cd95b6e 100644
--- a/yarns/lib.py
+++ b/yarns/lib.py
@@ -1,4 +1,4 @@
-# Copyright 2017 Lars Wirzenius
+# Copyright 2017-2018 Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -153,3 +153,68 @@ def delete(url, token):
     }
     r = requests.delete(url, headers=headers, verify=False)
     return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text
+
+
+def dict_diff(a, b):
+    if not isinstance(a, dict):
+        return 'first value is not a dict'
+    if not isinstance(b, dict):
+        return 'second value is not a dict'
+
+    delta = []
+
+    for key in a:
+        if key not in b:
+            delta.append('second does not have key {}'.format(key))
+        elif isinstance(a[key], dict):
+            delta2 = dict_diff(a[key], b[key])
+            if delta2 is not None:
+                delta.append('key {}: dict values differ:'.format(key))
+                delta.append(delta2)
+        elif isinstance(a[key], list):
+            delta2 = list_diff(a[key], b[key])
+            if delta2 is not None:
+                delta.append('key {}: list values differ:'.format(key))
+                delta.append(delta2)
+        elif a[key] != b[key]:
+            delta.append('key {}: values differ'.format(key))
+            delta.append('  first value : {!r}'.format(a[key]))
+            delta.append('  second value: {!r}'.format(b[key]))
+
+    for key in b:
+        if key not in a:
+            delta.append('first does not have key {}'.format(key))
+
+    if delta:
+        return '\n'.join(delta)
+    return None
+
+
+def list_diff(a, b):
+    if not isinstance(a, list):
+        return 'first value is not a list'
+    if not isinstance(b, list):
+        return 'second value is not a list'
+
+    delta = []
+
+    for i in range(len(a)):
+        if i >= len(b):
+            delta.append('second list is shorter than first')
+            break
+        elif isinstance(a[i], dict):
+            delta2 = dict_diff(a[i], b[i])
+            if delta2 is not None:
+                delta.append('item {}: items are different dicts'.format(i))
+                delta.append(delta2)
+        elif a[i] != b[i]:
+            delta.append('item %d: values differ'.format(i))
+            delta.append('  first value : {!r}'.format(a[i]))
+            delta.append('  second value: {!r}'.format(b[i]))
+
+    if len(a) < len(b):
+        delta.append('first list is shorter than second')
+        
+    if delta:
+        return '\n'.join(delta)
+    return None