Merge branch 'liw/blobs'

12 files changed, 427 insertions, 5 deletions

diff --git a/NEWS b/NEWS
index 5b0bef9..1e75efe 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 Version 0.19+git, not yet released
 ----------------------------------
 
+* A blob service has been added. It is so simple you will weep.
 
 Version 0.19, released 2017-11-29
 ----------------------------------
diff --git a/blob_service.py b/blob_service.py
new file mode 100644
index 0000000..44141f6
--- /dev/null
+++ b/blob_service.py
@@ -0,0 +1,122 @@
+#!/usr/bin/python3
+# Copyright (C) 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+import logging.handlers
+import os
+import sys
+
+
+import apifw
+import slog
+import yaml
+
+
+import ick2
+
+
+transactions = slog.Counter()
+
+
+def counter():
+    return 'HTTP transaction {}'.format(transactions.increment())
+
+
+def dict_logger(log, stack_info=None):
+    ick2.log.log(exc_info=stack_info, **log)
+
+
+default_config = {
+    'token-public-key': None,
+    'token-audience': None,
+    'token-issuer': None,
+    'log': [],
+    'blobdir': None,
+}
+
+
+def load_config(filename, defconf):
+    conf = yaml.safe_load(open(filename, 'r'))
+    actual_config = dict(defconf)
+    actual_config.update(conf)
+    return actual_config
+
+
+def check_config(config, musthave):
+    for key in config:
+        if key not in musthave:
+            raise Exception('Config %s is not known' % key)
+    for key in musthave:
+        if config.get(key) is None:
+            raise Exception('Config %s must not be None' % key)
+
+
+def main():
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    handler = logging.StreamHandler(sys.stderr)
+    logger.addHandler(handler)
+    logging.info('Starting blob service main program')
+
+    try:
+
+        config_filename = os.environ.get('BLOB_SERVICE_CONFIG')
+        logging.info('config filename %r', config_filename)
+        if not config_filename:
+            logging.error('no BLOB_SERVICE_CONFIG in environment')
+            raise Exception('No BLOB_SERVICE_CONFIG defined in environment')
+
+        logging.info('reading config from %r', config_filename)
+        config = load_config(config_filename, default_config)
+
+        logging.info('config is %r', config)
+        ick2.setup_logging(config)
+        check_config(config, default_config)
+
+        ick2.log.log('info', msg_text='Blob service starts', config=config)
+
+        api = ick2.BlobAPI()
+        ick2.log.log('info', msg_text='created BlobAPI')
+
+        api.set_blob_directory(config['blobdir'])
+        ick2.log.log(
+            'info', msg_text='called BlobAPI.set_blob_directory')
+
+        application = apifw.create_bottle_application(
+            api, counter, dict_logger, config)
+        ick2.log.log('info', msg_text='called apifw.create_bottle_application')
+
+        return application
+    except SystemExit:
+        raise
+    except BaseException as e:
+        logging.error(str(e))
+        ick2.log.log(
+            'error', msg_text='Uncaught  exception', exception=str(e),
+            exc_info=True)
+        sys.exit(1)
+
+
+app = main()
+
+# If we are running this program directly with Python, and not via
+# gunicorn, we can use the Bottle built-in debug server, which can
+# make some things easier to debug.
+
+if __name__ == '__main__':
+    print('running in debug mode')
+    app.run(host='127.0.0.1', port=12765)
diff --git a/ick2/__init__.py b/ick2/__init__.py
index 9b4d9e3..1fffe31 100644
--- a/ick2/__init__.py
+++ b/ick2/__init__.py
@@ -41,3 +41,5 @@ from .workerapi import WorkerAPI
 from .controllerapi import (
     ControllerAPI,
 )
+from .blobapi import BlobAPI
+from .blob_store import BlobStore
diff --git a/ick2/blob_store.py b/ick2/blob_store.py
new file mode 100644
index 0000000..7d02cbf
--- /dev/null
+++ b/ick2/blob_store.py
@@ -0,0 +1,51 @@
+# Copyright (C) 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+
+
+import ick2
+
+
+class BlobStore:
+
+    def __init__(self):
+        self._blobdir = None
+
+    def get_blob_directory(self):
+        return self._blobdir
+
+    def set_blob_directory(self, dirname):
+        self._blobdir = dirname
+        if not os.path.exists(self._blobdir):
+            os.makedirs(self._blobdir)
+
+    def store_blob(self, name, blob):
+        filename = self._blob_filename(name)
+        ick2.log.log(
+            'trace', msg_text='store_blob', name=name, filename=filename)
+        with open(filename, 'wb') as f:
+            f.write(blob)
+
+    def get_blob(self, name):
+        filename = self._blob_filename(name)
+        ick2.log.log(
+            'trace', msg_text='get_blob', name=name, filename=filename)
+        with open(filename, 'rb') as f:
+            return f.read()
+
+    def _blob_filename(self, name):
+        return os.path.join(self._blobdir, name)
diff --git a/ick2/blobapi.py b/ick2/blobapi.py
new file mode 100644
index 0000000..1772a27
--- /dev/null
+++ b/ick2/blobapi.py
@@ -0,0 +1,62 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import ick2
+
+
+class BlobAPI:
+
+    def __init__(self):
+        self._blobs = ick2.BlobStore()
+
+    def get_blob_directory(self):
+        return self._blobs.get_blob_directory()
+
+    def set_blob_directory(self, dirname):
+        self._blobs.set_blob_directory(dirname)
+
+    def find_missing_route(self, missing_path):
+        path = '/blobs/<blob_id>'
+        return [
+            {
+                'method': 'PUT',
+                'path': path,
+                'callback': self.put_blob,
+            },
+            {
+                'method': 'GET',
+                'path': path,
+                'callback': self.get_blob,
+            },
+        ]
+
+    def put_blob(self, content_type, body, blob_id, **kwargs):
+        ick2.log.log(
+            'info', msg_text='blob uploaded', blob_id=blob_id, kwargs=kwargs)
+        self._blobs.store_blob(blob_id, body)
+        return ick2.OK('')
+
+    def get_blob(self, content_type, body, blob_id, **kwargs):
+        ick2.log.log(
+            'info', msg_text='blob requested', blob_id=blob_id, kwargs=kwargs)
+        try:
+            blob = self._blobs.get_blob(blob_id)
+        except EnvironmentError as e:
+            return ick2.not_found(str(e))
+        headers = {
+            'Content-Type': 'application/octet-stream',
+        }
+        return ick2.OK(blob, headers=headers)
diff --git a/ick2/responses.py b/ick2/responses.py
index 1ceae52..f0c5c8f 100644
--- a/ick2/responses.py
+++ b/ick2/responses.py
@@ -25,10 +25,11 @@ def response(status_code, body, headers):  # pragma: no cover
     return apifw.Response(obj)
 
 
-def OK(body):  # pragma: no cover
-    headers = {
-        'Content-Type': 'application/json',
-    }
+def OK(body, headers=None):  # pragma: no cover
+    if headers is None:
+        headers = {}
+    elif 'Content-Type' not in headers:
+        headers['Content-Type'] = 'application/json'
     return response(apifw.HTTP_OK, body, headers)
 
 
diff --git a/run-blob-service-debug b/run-blob-service-debug
new file mode 100755
index 0000000..77bade4
--- /dev/null
+++ b/run-blob-service-debug
@@ -0,0 +1,35 @@
+#!/bin/sh
+# Copyright 2017 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eu
+
+scopes="
+uapi_blobs_id_put
+uapi_blobs_id_get
+"
+
+./generate-rsa-key t.key
+./create-token < t.key "$scopes" > t.token
+cat <<EOF > t.yaml
+log:
+- filename: t.log
+token-issuer: localhost
+token-audience: localhost
+token-public-key: $(cat t.key.pub)
+blobdir: t.blobs
+EOF
+
+BLOB_SERVICE_CONFIG=t.yaml python3 blob_service.py
diff --git a/without-tests b/without-tests
index 6b58af3..f7425bf 100644
--- a/without-tests
+++ b/without-tests
@@ -1,5 +1,7 @@
 ick2/__init__.py
 ick2/apibase.py
+ick2/blobapi.py
+ick2/blob_store.py
 ick2/buildsapi.py
 ick2/exceptions.py
 ick2/logapi.py
diff --git a/yarns/700-blob-service.yarn b/yarns/700-blob-service.yarn
new file mode 100644
index 0000000..1fd0c06
--- /dev/null
+++ b/yarns/700-blob-service.yarn
@@ -0,0 +1,52 @@
+<!--
+
+Copyright 2017 Lars Wirzenius
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+-->
+
+# Blob service
+
+This scenario tests the blob service API to store and retrieve blobs.
+At this stage the blob service is the simplest possible; so simple, in
+fact, it will certainly change in the future.
+
+    SCENARIO blob service
+
+Set up the blob service.
+
+    GIVEN an RSA key pair for token signing
+    AND blob service config uses blobs at the blob directory
+    AND an access token for user with scopes
+    ... uapi_blobs_id_put
+    ... uapi_blobs_id_get
+    AND a running blob service
+
+Try to get a non-existent blob. It should result in an error.
+
+    WHEN user retrieves /blobs/cake from blob service
+    THEN result has status code 404
+
+Create and store a blob, retrieve it and verify we get it back intack.
+
+    WHEN user creates a blob named cake with random data
+    AND user sends blob cake to blob service as /blobs/cake
+    THEN result has status code 200
+    
+    WHEN user retrieves /blobs/cake from blob service
+    THEN result has status code 200
+    AND body is the same as the blob cake
+
+    FINALLY stop blob service
diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn
index d08cbc9..afddb3b 100644
--- a/yarns/900-implements.yarn
+++ b/yarns/900-implements.yarn
@@ -19,6 +19,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Scenario step implementations (local and remote instances)
 
+## Data file creation
+
+    IMPLEMENTS WHEN user creates a blob named (\S+) with random data
+    filename = get_next_match()
+    n = 16
+    blob = os.urandom(n)
+    write(filename, blob)
+
 ## HTTP requests of various kinds
 
     IMPLEMENTS WHEN (\S+) makes request GET (\S+)
@@ -32,6 +40,17 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     vars['headers'] = headers
     vars['body'] = body
 
+    IMPLEMENTS WHEN (\S+) retrieves (\S+) from blob service
+    user = get_next_match()
+    path = get_next_match()
+    token = get_token(user)
+    url = vars['bsurl']
+    status, content_type, headers, body = get_blob(url + path, token)
+    vars['status_code'] = status
+    vars['content_type'] = content_type
+    vars['headers'] = headers
+    vars['body'] = body.encode('hex')
+
     IMPLEMENTS WHEN (\S+) makes request GET (\S+) with an invalid token
     user = get_next_match()
     path = get_next_match()
@@ -69,7 +88,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     vars['status_code'] = status
     vars['content_type'] = content_type
     vars['headers'] = headers
-    vars['body'] = body
+    vars['body'] = body.encode('hex')
 
     IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with a valid token and body (.+)
     user = get_next_match()
@@ -86,6 +105,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     vars['headers'] = headers
     vars['body'] = body
 
+    IMPLEMENTS WHEN (\S+) sends blob (\S+) to blob service as (\S+)
+    user = get_next_match()
+    filename = get_next_match()
+    path = get_next_match()
+    body = cat(filename)
+    token = get_token(user)
+    url = vars['bsurl']
+    status, content_type, headers, body = put_blob(url + path, body, token)
+    vars['status_code'] = status
+
     IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with an invalid token
     user = get_next_match()
     path = get_next_match()
@@ -130,6 +159,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     actual = vars['body']
     assertEqual(expected, actual)
 
+    IMPLEMENTS THEN body is the same as the blob (\S+)
+    filename = get_next_match()
+    blob = cat(filename)
+    body = vars['body'].decode('hex')
+    assertEqual(body, blob)
+
     IMPLEMENTS THEN version in body matches version from setup.py
     body = vars['body']
     obj = json.loads(body)
diff --git a/yarns/900-local.yarn b/yarns/900-local.yarn
index 409a8e6..89652de 100644
--- a/yarns/900-local.yarn
+++ b/yarns/900-local.yarn
@@ -100,3 +100,45 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     name = get_next_match()
     filename = os.path.join(vars['statedir'], 'workers', name + '.yaml')
     assertTrue(os.path.exists(filename))
+
+## Start and stop blob service
+
+    IMPLEMENTS GIVEN blob service config uses (\S+) at the blob directory
+    vars['blobdir'] = get_next_match()
+
+    IMPLEMENTS GIVEN a running blob service
+    import os, time, cliapp, yaml
+    vars['blob_service.log'] = 'blob_service.log'
+    vars['gunicorn3_bs.log'] = 'gunicorn3_bs.log'
+    vars['bsport'] = random_free_port()
+    vars['bsurl'] = 'http://127.0.0.1:{}'.format(vars['bsport'])
+    config = {
+        'token-issuer': vars['issuer'],
+        'token-audience': vars['audience'],
+        'token-public-key': cat('token.key.pub'),
+        'log': [
+            {
+                'filename': vars['blob_service.log'],
+            },
+        ],
+        'blobdir': vars['blobdir'],
+    }
+    env = dict(os.environ)
+    env['BLOB_SERVICE_CONFIG'] = 'blob_service.yaml'
+    yaml.safe_dump(config, open('blob_service.yaml', 'w'))
+    argv = [
+        'gunicorn3',
+        '--daemon',
+        '--bind', '127.0.0.1:{}'.format(vars['bsport']),
+        '--log-file', vars['gunicorn3_bs.log'],
+        '--log-level', 'debug',
+        '-p', 'bspid',
+        'blob_service:app',
+    ]
+    cliapp.runcmd(argv, env=env)
+    vars['bspid'] = int(cat('bspid'))
+    wait_for_port(vars['bsport'])
+
+    IMPLEMENTS FINALLY stop blob service
+    import os, signal
+    os.kill(vars['bspid'], signal.SIGTERM)
diff --git a/yarns/lib.py b/yarns/lib.py
index 486757c..6575e6d 100644
--- a/yarns/lib.py
+++ b/yarns/lib.py
@@ -104,6 +104,14 @@ def get(url, token):
     return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text
 
 
+def get_blob(url, token):
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+    }
+    r = requests.get(url, headers=headers, verify=False)
+    return r.status_code, r.headers['Content-Type'], dict(r.headers), r.content
+
+
 def post(url, body_text, token):
     headers = {
         'Authorization': 'Bearer {}'.format(token),
@@ -122,6 +130,15 @@ def put(url, body_text, token):
     return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text
 
 
+def put_blob(url, body, token):
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+        'Content-Type': 'application/octet-stream',
+    }
+    r = requests.put(url, headers=headers, data=body, verify=False)
+    return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text
+
+
 def delete(url, token):
     headers = {
         'Authorization': 'Bearer {}'.format(token),