Add: ControllerAPI, ControllerState

16 files changed, 1004 insertions, 5 deletions

diff --git a/check b/check
index f8992f4..269c1b9 100755
--- a/check
+++ b/check
@@ -2,7 +2,7 @@
 
 set -eu
 
-python -m CoverageTestRunner --ignore-missing-from=without-tests ick2lib
+python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2
 
 yarn yarns/*.yarn \
      --shell python2 \
diff --git a/create-token b/create-token
new file mode 100755
index 0000000..e2e9fbb
--- /dev/null
+++ b/create-token
@@ -0,0 +1,44 @@
+#!/usr/bin/python3
+# Copyright (C) 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import sys
+import time
+
+import Crypto.PublicKey.RSA
+
+import apifw
+
+
+filename = sys.argv[1]
+iss = sys.argv[2]
+aud = sys.argv[3]
+scopes = ' '.join(sys.argv[4].split())
+
+key_text = open(filename, 'r').read()
+key = Crypto.PublicKey.RSA.importKey(key_text)
+
+now = time.time()
+claims = {
+    'iss': iss,
+    'sub': 'subject-uuid',
+    'aud': aud,
+    'exp': now + 3600,
+    'scope': scopes,
+}
+
+token = apifw.create_token(claims, key)
+sys.stdout.write(token.decode('ascii'))
diff --git a/generate-rsa-key b/generate-rsa-key
new file mode 100755
index 0000000..d73ba0d
--- /dev/null
+++ b/generate-rsa-key
@@ -0,0 +1,34 @@
+#!/usr/bin/python3
+# Copyright (C) 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import sys
+
+import Crypto.PublicKey.RSA
+
+
+RSA_KEY_BITS = 4096  # A nice, currently safe length
+
+key = Crypto.PublicKey.RSA.generate(RSA_KEY_BITS)
+
+filename = sys.argv[1]
+
+def write(filename, byts):
+    with open(filename, 'w') as f:
+        f.write(byts.decode('ascii'))
+
+write(filename, key.exportKey('PEM'))
+write(filename + '.pub', key.exportKey('OpenSSH'))
diff --git a/ick2/__init__.py b/ick2/__init__.py
new file mode 100644
index 0000000..bf349aa
--- /dev/null
+++ b/ick2/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (C) 2017  Lars Wirzenius
+
+
+from .logging import setup_logging, log
+from .state import ControllerState, NotFound
+from .controllerapi import ControllerAPI
diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py
new file mode 100644
index 0000000..c2b3213
--- /dev/null
+++ b/ick2/controllerapi.py
@@ -0,0 +1,120 @@
+# Copyright (C) 2017  Lars Wirzenius
+
+
+import glob
+import os
+
+
+import apifw
+import yaml
+
+
+import ick2
+
+
+class ControllerAPI:
+
+    def __init__(self):
+        self._state = ick2.ControllerState()
+
+    def get_state_directory(self):
+        return self._state.get_state_directory()
+
+    def set_state_directory(self, dirname):
+        self._state.set_state_directory(dirname)
+
+    def load_projects(self):
+        return self._state.load_projects()
+
+    def find_missing_route(self, path):  # pragma: no cover
+        return [
+            {
+                'method': 'POST',
+                'path': '/projects',
+                'callback': self.post_callback(self.post_projects),
+            },
+            {
+                'method': 'GET',
+                'path': '/projects',
+                'callback': self.get_callback(self.get_projects),
+            },
+            {
+                'method': 'GET',
+                'path': '/projects/<project>',
+                'callback': self.get_callback(self.get_project),
+            },
+            {
+                'method': 'PUT',
+                'path': '/projects/<project>',
+                'callback': self.put_callback(self.put_projects),
+            },
+            {
+                'method': 'DELETE',
+                'path': '/projects/<project>',
+                'callback': self.get_callback(self.delete_projects),
+            },
+        ]
+
+    def get_callback(self, callback):  # pragma: no cover
+        def wrapper(content_type, body, **kwargs):
+            try:
+                body = callback(**kwargs)
+            except ick2.NotFound as e:
+                return apifw.Response({
+                    'status': apifw.HTTP_NOT_FOUND,
+                    'body': str(e),
+                    'headers': [],
+                })
+            return apifw.Response({
+                'status': apifw.HTTP_OK,
+                'body': body,
+                'headers': {
+                    'Content-Type': 'application/json',
+                },
+            })
+        return wrapper
+
+    def post_callback(self, callback):  # pragma: no cover
+        def wrapper(content_type, body):
+            body = callback(body)
+            ick2.log.log('trace', msg_text='returned body', body=repr(body))
+            return apifw.Response({
+                'status': apifw.HTTP_CREATED,
+                'body': body,
+                'headers': {
+                    'Content-Type': 'application/json',
+                },
+            })
+        return wrapper
+
+    def put_callback(self, callback):  # pragma: no cover
+        def wrapper(content_type, body, **kwargs):
+            body = callback(body, **kwargs)
+            ick2.log.log('trace', msg_text='returned body', body=repr(body))
+            return apifw.Response({
+                'status': apifw.HTTP_OK,
+                'body': body,
+                'headers': {
+                    'Content-Type': 'application/json',
+                },
+            })
+        return wrapper
+
+    def get_projects(self):
+        return {
+            'projects': self._state.get_projects(),
+        }
+
+    def get_project(self, project=None):
+        assert project is not None
+        return self._state.get_project(project)
+
+    def post_projects(self, project):
+        return self._state.add_project(project)
+
+    def put_projects(self, body, project=None):
+        assert project is not None
+        return self._state.update_project(project, body)
+
+    def delete_projects(self, project):
+        self._state.remove_project(project)
diff --git a/ick2/controllerapi_tests.py b/ick2/controllerapi_tests.py
new file mode 100644
index 0000000..3277526
--- /dev/null
+++ b/ick2/controllerapi_tests.py
@@ -0,0 +1,100 @@
+# Copyright (C) 2017  Lars Wirzenius
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+
+import ick2
+
+
+class ControllerAPITests(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.statedir = os.path.join(self.tempdir, 'state/dir')
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def create_api(self):
+        api = ick2.ControllerAPI()
+        api.set_state_directory(self.statedir)
+        return api
+
+    def test_has_no_state_directory_initially(self):
+        api = ick2.ControllerAPI()
+        statedir = api.get_state_directory()
+        self.assertTrue(statedir is None)
+
+    def test_sets_and_creates_state_directory(self):
+        api = self.create_api()
+        statedir = api.get_state_directory()
+        self.assertEqual(statedir, self.statedir)
+        self.assertTrue(os.path.exists(statedir))
+    
+    def test_has_not_projects_initially(self):
+        api = self.create_api()
+        self.assertEqual(api.get_projects(), {'projects': []})
+
+    def test_creates_project(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        api = self.create_api()
+        self.assertEqual(api.post_projects(project), project)
+        self.assertEqual(api.get_projects(), {'projects': [project]})
+
+    def test_loads_projects_from_state_directory(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        api = self.create_api()
+        api.post_projects(project)
+
+        api2 = self.create_api()
+        api2.load_projects()
+        self.assertEqual(api2.get_projects(), {'projects': [project]})
+
+    def test_gets_named_project(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        api = self.create_api()
+        api.post_projects(project)
+        self.assertEqual(api.get_project('foo'), project)
+
+    def test_updates_named_project(self):
+        project_v1 = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        project_v2 = dict(project_v1)
+        project_v2['shell_steps'] = ['build it using magic']
+        api = self.create_api()
+        api.post_projects(project_v1)
+        updated = api.put_projects(project_v2, project='foo')
+        self.assertEqual(updated, project_v2)
+        self.assertEqual(api.get_project('foo'), project_v2)
+
+    def test_deletes_named_project(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        api = self.create_api()
+        api.post_projects(project)
+        api.delete_projects('foo')
+        self.assertEqual(api.get_projects(), {'projects': []})
+        with self.assertRaises(ick2.NotFound):
+            api.get_project('foo')
+
+    def test_raises_errro_deleting_missing_project(self):
+        api = self.create_api()
+        with self.assertRaises(ick2.NotFound):
+            api.delete_projects('foo')
diff --git a/ick2/logging.py b/ick2/logging.py
new file mode 100644
index 0000000..729778d
--- /dev/null
+++ b/ick2/logging.py
@@ -0,0 +1,57 @@
+# Copyright (C) 2017  Lars Wirzenius
+
+
+import slog
+
+
+def drop_get_message(log_obj):
+    # These are useless and annoying in gunicorn log messages.
+    if 'getMessage' in log_obj:
+        del log_obj['getMessage']
+    return log_obj
+
+
+# We are probably run under gunicorn, which sets up logging via the
+# logging library. Hijack that so actual logging happens via the slog
+# library. For this, we need to know the logger names gunicorn uses.
+gunicorn_loggers = ['gunicorn.access', 'gunicorn.error']
+
+
+# This sets up a global log variable that doesn't actually log
+# anything anywhere. This is useful so that code can unconditionally
+# call log.log(...) from anywhere. See setup_logging() for setting up
+# actual logging to somewhere persistent.
+
+log = slog.StructuredLog()
+log.add_log_writer(slog.NullSlogWriter(), slog.FilterAllow())
+log.add_log_massager(drop_get_message)
+slog.hijack_logging(log, logger_names=gunicorn_loggers)
+
+
+def setup_logging(config):
+    if 'log' in config:
+        assert isinstance(config['log'], list)
+        for target in config.get('log', []):
+            setup_logging_to_target(target)
+
+
+def setup_logging_to_target(target):
+    rule = get_filter_rules(target)
+    if 'filename' in target:
+        setup_logging_to_file(target, rule)
+    else:
+        raise Exception('Do not understand logging target %r' % target)
+
+
+def get_filter_rules(target):
+    if 'filter' in target:
+        return slog.construct_log_filter(target['filter'])
+    return slog.FilterAllow()
+
+
+def setup_logging_to_file(target, rule):
+    writer = slog.FileSlogWriter()
+    writer.set_filename(target['filename'])
+    if 'max_bytes' in target:
+        writer.set_max_file_size(target['max_bytes'])
+    log.add_log_writer(writer, rule)
diff --git a/ick2/state.py b/ick2/state.py
new file mode 100644
index 0000000..3d4199f
--- /dev/null
+++ b/ick2/state.py
@@ -0,0 +1,83 @@
+# Copyright (C) 2017  Lars Wirzenius
+
+
+import glob
+import os
+
+
+import apifw
+import yaml
+
+
+import ick2
+
+
+class ControllerState:
+
+    def __init__(self):
+        self._statedir = None
+
+    def get_state_directory(self):
+        return self._statedir
+
+    def set_state_directory(self, dirname):
+        self._statedir = dirname
+        if not os.path.exists(self._statedir):
+            os.makedirs(self._statedir)
+
+    def get_project_directory(self):
+        return os.path.join(self.get_state_directory(), 'projects')
+
+    def get_project_filename(self, project_name):
+        return os.path.join(
+            self.get_project_directory(), project_name + '.yaml')
+
+    def load_projects(self):
+        assert self._statedir is not None
+        projects = []
+        dirname = self.get_project_directory()
+        for filename in glob.glob(dirname + '/*.yaml'):
+            obj = self.load_project(filename)
+            projects.append(obj)
+        return projects
+
+    def load_project(self, filename):
+        with open(filename, 'r') as f:
+            return yaml.safe_load(f)
+
+    def get_projects(self):
+        return self.load_projects()
+
+    def get_project(self, project):
+        filename = self.get_project_filename(project)
+        if os.path.exists(filename):
+            return self.load_project(filename)
+        raise NotFound()
+
+    def add_project(self, project):
+        filename = self.get_project_filename(project['project'])
+        dirname = os.path.dirname(filename)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(filename, 'w') as f:
+            yaml.safe_dump(project, stream=f)
+        return project
+
+    def update_project(self, project, body):
+        filename = self.get_project_filename(project)
+        dirname = os.path.dirname(filename)
+        with open(filename, 'w') as f:
+            yaml.safe_dump(body, stream=f)
+        return body
+
+    def remove_project(self, project):
+        filename = self.get_project_filename(project)
+        if not os.path.exists(filename):
+            raise NotFound()
+        os.remove(filename)
+
+
+class NotFound(Exception):
+
+    def __init__(self):
+        super().__init__('Resource not found')
diff --git a/ick2/state_tests.py b/ick2/state_tests.py
new file mode 100644
index 0000000..098ad2d
--- /dev/null
+++ b/ick2/state_tests.py
@@ -0,0 +1,100 @@
+# Copyright (C) 2017  Lars Wirzenius
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+
+import ick2
+
+
+class ControllerStateTests(unittest.TestCase):
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.statedir = os.path.join(self.tempdir, 'state/dir')
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    def create_state(self):
+        state = ick2.ControllerState()
+        state.set_state_directory(self.statedir)
+        return state
+
+    def test_has_no_state_directory_initially(self):
+        state = ick2.ControllerState()
+        self.assertTrue(state.get_state_directory() is None)
+
+    def test_sets_and_creates_state_directory(self):
+        state = self.create_state()
+        statedir = state.get_state_directory()
+        self.assertEqual(statedir, self.statedir)
+        self.assertTrue(os.path.exists(statedir))
+    
+    def test_has_not_projects_initially(self):
+        state = self.create_state()
+        self.assertEqual(state.get_projects(), [])
+
+    def test_creates_project(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        state = self.create_state()
+        state.add_project(project)
+        self.assertEqual(state.get_projects(), [project])
+        self.assertTrue(os.path.exists(state.get_project_filename('foo')))
+
+    def test_loads_projects_from_state_directory(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        state = self.create_state()
+        state.add_project(project)
+
+        state2 = self.create_state()
+        state2.load_projects()
+        self.assertEqual(state2.get_projects(), [project])
+
+    def test_gets_named_project(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        state = self.create_state()
+        state.add_project(project)
+        self.assertEqual(state.get_project('foo'), project)
+
+    def test_updates_named_project(self):
+        project_v1 = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        project_v2 = dict(project_v1)
+        project_v2['shell_steps'] = ['build it using magic']
+        state = self.create_state()
+        state.add_project(project_v1)
+        updated = state.update_project('foo', project_v2)
+        self.assertEqual(updated, project_v2)
+        self.assertEqual(state.get_project('foo'), project_v2)
+
+    def test_deletes_named_project(self):
+        project = {
+            'project': 'foo',
+            'shell_steps': ['build'],
+        }
+        state = self.create_state()
+        state.add_project(project)
+        state.remove_project('foo')
+        self.assertEqual(state.get_projects(), [])
+        with self.assertRaises(ick2.NotFound):
+            state.get_project('foo')
+
+    def test_raises_error_deleting_missing_project(self):
+        state = self.create_state()
+        with self.assertRaises(ick2.NotFound):
+            state.remove_project('foo')
diff --git a/ick_controller.py b/ick_controller.py
new file mode 100644
index 0000000..e201a40
--- /dev/null
+++ b/ick_controller.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+# Copyright (C) 2017  Lars Wirzenius
+
+
+import os
+
+
+import apifw
+import Crypto.PublicKey.RSA
+import slog
+import yaml
+
+
+import ick2
+
+
+transactions = slog.Counter()
+def counter():
+    return 'HTTP transaction {}'.format(transactions.increment())
+
+
+def dict_logger(log, stack_info=None):
+    ick2.log.log(exc_info=stack_info, **log)
+
+
+default_config = {
+    'token-public-key': None,
+    'token-audience': None,
+    'token-issuer': None,
+    'log': [],
+    'statedir': None,
+}
+
+
+def load_config(filename, default_config):
+    config = yaml.safe_load(open(filename, 'r'))
+    actual_config = dict(default_config)
+    actual_config.update(config)
+    return actual_config
+
+
+config_filename = os.environ.get('ICK_CONTROLLER_CONFIG')
+if not config_filename:
+    raise Exception('No ICK_CONTROLLER_CONFIG defined in environment')
+config = load_config(config_filename, default_config)
+ick2.setup_logging(config)
+ick2.log.log('info', msg_text='Ick2 controller starts', config=config)
+api = ick2.ControllerAPI()
+api.set_state_directory(config['statedir'])
+api.load_projects()
+app = apifw.create_bottle_application(api, counter, dict_logger, config)
+
+# If we are running this program directly with Python, and not via
+# gunicorn, we can use the Bottle built-in debug server, which can
+# make some things easier to debug.
+
+if __name__ == '__main__':
+    print('running in debug mode')
+    app.run(host='127.0.0.1', port=12765)
diff --git a/run-debug b/run-debug
new file mode 100755
index 0000000..fdd1ce8
--- /dev/null
+++ b/run-debug
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -eu
+
+scopes="
+uapi_projects_get
+"
+
+./generate-rsa-key t.key
+./create-token t.key issuer audience "$scopes" > t.token
+cat <<EOF > t.yaml
+log:
+- filename: t.log
+token-issuer: issuer
+token-audience: audience
+token-public-key: $(cat t.key.pub)
+statedir: t.state
+EOF
+
+ICK_CONTROLLER_CONFIG=t.yaml python3 ick_controller.py
diff --git a/without-tests b/without-tests
index bb41680..4950c95 100644
--- a/without-tests
+++ b/without-tests
@@ -1,5 +1,4 @@
-ick2lib/__init__.py
-ick2lib/apiservice.py
-ick2lib/app.py
-ick2lib/version.py
+ick2/__init__.py
+ick2/logging.py
+
 
diff --git a/yarns/000.yarn b/yarns/000.yarn
new file mode 100644
index 0000000..d022d12
--- /dev/null
+++ b/yarns/000.yarn
@@ -0,0 +1,11 @@
+---
+title: Ick2 integration tests
+author: Lars Wirzenius
+version: work in progress
+...
+
+
+# Introduction
+
+This is a set of integration tests for Ick2, a continuous integration
+system. Written for execution by yarn.
diff --git a/yarns/100-projects.yarn b/yarns/100-projects.yarn
new file mode 100644
index 0000000..315116f
--- /dev/null
+++ b/yarns/100-projects.yarn
@@ -0,0 +1,132 @@
+# Controller project management
+
+The Ick2 controller manages information about projects. Projects are
+things the controller builds. A project is described by a resource
+like this:
+
+    EXAMPLE project resource
+    {
+        "name": "ick2-website",
+        "shell_commands": [
+            "git clone git://git.liw.fi/ick2-website src",
+            "cd src && ikiwiki --setup ikiwiki.setup",
+            "cd html && rsync -a --delete . www-data@www.example.com:/srv/http/ick2/."
+        ]
+    }
+
+In other words, there are two things that define a project:
+
+* The name. This is used for referreing to the project in the API.
+* A sequence of shell commands to be run to build the project. At this
+  point Ick2 does not know about git repositories, so the shell
+  commands need to do the cloning explicitly; this will obviously have
+  to be changed later on. Note also that each string in the list is
+  run by a new shell process, and that the current working directory,
+  or environment variables, do not get preserved. Ick2 will create a
+  new, empty directory for running the commands.
+
+## Managing projects
+
+First we test the controller API for managing projects, without
+building them. We start by starting an instance of the controller.
+
+    SCENARIO managing projects
+    GIVEN an RSA key pair for token signing
+    AND an access token for scopes 
+    ... uapi_projects_get
+    ... uapi_projects_post
+    ... uapi_projects_id_get
+    ... uapi_projects_id_put
+    ... uapi_projects_id_delete
+    AND controller config uses statedir at the state directory
+    AND a running ick controller
+
+    WHEN user makes request GET /projects
+    THEN result has status code 200
+    AND body matches { "projects": [] }
+
+    WHEN user makes request POST /projects
+    ... {
+    ...     "project": "website",
+    ...     "shell_steps": [
+    ...         "git clone git://repo src",
+    ...         "mkdir html",
+    ...         "ikiwiki src html"
+    ...     ]
+    ... }
+    THEN result has status code 201
+    AND body matches 
+    ... {
+    ...     "project": "website",
+    ...     "shell_steps": [
+    ...         "git clone git://repo src",
+    ...         "mkdir html",
+    ...         "ikiwiki src html"
+    ...     ]
+    ... }
+    AND controller state directory contains project website
+
+    WHEN user makes request GET /projects
+    THEN result has status code 200
+    AND body matches 
+    ... {
+    ...     "projects": [
+    ...         {
+    ...             "project": "website",
+    ...             "shell_steps": [
+    ...                 "git clone git://repo src",
+    ...                 "mkdir html",
+    ...                 "ikiwiki src html"
+    ...             ]
+    ...         }
+    ...     ]
+    ... }
+
+    WHEN user stops ick controller
+    GIVEN a running ick controller
+    WHEN user makes request GET /projects/website
+    THEN result has status code 200
+    AND body matches 
+    ... {
+    ...     "project": "website",
+    ...     "shell_steps": [
+    ...         "git clone git://repo src",
+    ...         "mkdir html",
+    ...         "ikiwiki src html"
+    ...     ]
+    ... }
+
+    WHEN user makes request PUT /projects/website
+    ... {
+    ...     "project": "website",
+    ...     "shell_steps": [
+    ...         "build-it"
+    ...     ]
+    ... }
+    THEN result has status code 200
+    AND body matches 
+    ... {
+    ...     "project": "website",
+    ...     "shell_steps": [
+    ...         "build-it"
+    ...     ]
+    ... }
+    AND controller state directory contains project website
+
+    WHEN user makes request GET /projects/website
+    THEN result has status code 200
+    AND body matches 
+    ... {
+    ...     "project": "website",
+    ...     "shell_steps": [
+    ...         "build-it"
+    ...     ]
+    ... }
+
+    WHEN user makes request DELETE /projects/website
+    THEN result has status code 200
+    WHEN user makes request GET /projects/website
+    THEN result has status code 404
+
+
+    FINALLY stop ick controller
diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn
new file mode 100644
index 0000000..c9c5cf5
--- /dev/null
+++ b/yarns/900-implements.yarn
@@ -0,0 +1,140 @@
+# Scenario step implementations
+
+## Authentication setup
+
+    IMPLEMENTS GIVEN an RSA key pair for token signing
+    argv = [
+        os.path.join(srcdir, 'generate-rsa-key'),
+        'token.key',
+    ]
+    cliapp.runcmd(argv, stdout=None, stderr=None)
+
+    IMPLEMENTS GIVEN an access token for scopes (.+)
+    scopes = get_next_match()
+    vars['issuer'] = 'yarn-issuer'
+    vars['audience'] = 'yarn-audience'
+    argv = [
+        os.path.join(srcdir, 'create-token'),
+        'token.key',
+        vars['issuer'],
+        vars['audience'],
+        scopes,
+    ]
+    token = cliapp.runcmd(argv)
+    write('token.jwt', token)
+
+## Controller configuration
+
+    IMPLEMENTS GIVEN controller config uses (\S+) at the state directory
+    vars['statedir'] = get_next_match()
+
+## Start and stop the controller
+
+    IMPLEMENTS GIVEN a running ick controller
+    import os, time, cliapp, yaml
+    vars['controller.log'] = 'ick_controller.log'
+    vars['gunicorn3.log'] = 'gunicorn3.log'
+    vars['port'] = random_free_port()
+    vars['url'] = 'http://127.0.0.1:{}'.format(vars['port'])
+    config = {
+        'token-issuer': vars['issuer'],
+        'token-audience': vars['audience'],
+        'token-public-key': cat('token.key.pub'),
+        'log': [
+            {
+                'filename': vars['controller.log'],
+            },
+        ],
+        'statedir': vars['statedir'],
+    }
+    env = dict(os.environ)
+    env['ICK_CONTROLLER_CONFIG'] = 'ick_controller.yaml'
+    yaml.safe_dump(config, open('ick_controller.yaml', 'w'))
+    argv = [
+        'gunicorn3',
+        '--daemon',
+        '--bind', '127.0.0.1:{}'.format(vars['port']),
+        '--log-file', vars['gunicorn3.log'],
+        '--log-level', 'debug',
+        '-p', 'pid',
+        'ick_controller:app',
+    ]
+    cliapp.runcmd(argv, env=env)
+    vars['pid'] = int(cat('pid'))
+    wait_for_port(vars['port'])
+    
+    IMPLEMENTS WHEN user stops ick controller
+    import os, signal
+    os.kill(vars['pid'], signal.SIGTERM)
+    
+    IMPLEMENTS FINALLY stop ick controller
+    import os, signal
+    os.kill(vars['pid'], signal.SIGTERM)
+
+
+## HTTP requests of various kinds
+
+    IMPLEMENTS WHEN user makes request GET (\S+)
+    path = get_next_match()
+    token = cat('token.jwt')
+    url = vars['url']
+    status, content_type, body = get(url + path, token)
+    vars['status_code'] = status
+    vars['content_type'] = content_type
+    vars['body'] = body
+
+    IMPLEMENTS WHEN user makes request POST (\S+) (.+)
+    path = get_next_match()
+    body_text = get_next_match()
+    print('path', path)
+    print('body', body_text)
+    token = cat('token.jwt')
+    url = vars['url']
+    status, content_type, body = post(url + path, body_text, token)
+    vars['status_code'] = status
+    vars['content_type'] = content_type
+    vars['body'] = body
+
+    IMPLEMENTS WHEN user makes request PUT (\S+) (.+)
+    path = get_next_match()
+    body_text = get_next_match()
+    print('path', path)
+    print('body', body_text)
+    token = cat('token.jwt')
+    url = vars['url']
+    status, content_type, body = put(url + path, body_text, token)
+    vars['status_code'] = status
+    vars['content_type'] = content_type
+    vars['body'] = body
+
+    IMPLEMENTS WHEN user makes request DELETE (\S+)
+    path = get_next_match()
+    token = cat('token.jwt')
+    url = vars['url']
+    status, content_type, body = delete(url + path, token)
+    vars['status_code'] = status
+    vars['content_type'] = content_type
+    vars['body'] = body
+
+
+## HTTP response inspection
+
+    IMPLEMENTS THEN result has status code (\d+)
+    print(cat('token.jwt'))
+    expected = int(get_next_match())
+    assertEqual(expected, vars['status_code'])
+
+    IMPLEMENTS THEN body matches (.+)
+    expected_text = get_next_match()
+    expected = json.loads(expected_text)
+    print('actual body', repr(vars['body']))
+    actual = json.loads(vars['body'])
+    assertEqual(expected, actual)
+
+
+## Controller state inspection
+
+    IMPLEMENTS THEN controller state directory contains project (\S+)
+    name = get_next_match()
+    filename = os.path.join(vars['statedir'], 'projects', name + '.yaml')
+    assertTrue(os.path.exists(filename))
diff --git a/yarns/lib.py b/yarns/lib.py
new file mode 100644
index 0000000..4d305c7
--- /dev/null
+++ b/yarns/lib.py
@@ -0,0 +1,94 @@
+import errno
+import json
+import os
+import random
+import socket
+import sys
+import time
+
+import cliapp
+import requests
+
+from yarnutils import *
+
+
+srcdir = os.environ['SRCDIR']
+datadir = os.environ['DATADIR']
+vars = Variables(datadir)
+
+
+def random_free_port():
+    MAX = 1000
+    for i in range(MAX):
+        port = random.randint(1025, 2**15-1)
+        s = socket.socket()
+        try:
+            s.bind(('0.0.0.0', port))
+        except OSError as e:
+            if e.errno == errno.EADDRINUSE:
+                continue
+            print('cannot find a random free port')
+            raise
+        s.close()
+        break
+    print('picked port', port)
+    return port
+
+
+def wait_for_port(port):
+    MAX = 5
+    t = time.time()
+    while time.time() < t + MAX:
+        try:
+            s = socket.socket()
+            s.connect(('127.0.0.1', port))
+        except OSError as e:
+            raise
+        else:
+            return
+
+def write(filename, data):
+    with open(filename, 'w') as f:
+        f.write(data)
+
+
+def cat(filename):
+    MAX_CAT_WAIT = 5  # in seconds
+    t = time.time()
+    while time.time() < t + MAX_CAT_WAIT:
+        if os.path.exists(filename):
+            return open(filename, 'r').read()
+
+
+def get(url, token):
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+    }
+    r = requests.get(url, headers=headers)
+    return r.status_code, r.headers['Content-Type'], r.text
+
+
+def post(url, body_text, token):
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+        'Content-Type': 'application/json',
+    }
+    r = requests.post(url, headers=headers, data=body_text)
+    return r.status_code, r.headers['Content-Type'], r.text
+
+
+def put(url, body_text, token):
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+        'Content-Type': 'application/json',
+    }
+    r = requests.put(url, headers=headers, data=body_text)
+    return r.status_code, r.headers['Content-Type'], r.text
+
+
+def delete(url, token):
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+    }
+    r = requests.delete(url, headers=headers)
+    return r.status_code, r.headers['Content-Type'], r.text