Add: blob service

8 files changed, 286 insertions, 4 deletions

diff --git a/blob_service.py b/blob_service.py
new file mode 100644
index 0000000..44141f6
--- /dev/null
+++ b/blob_service.py
@@ -0,0 +1,122 @@
+#!/usr/bin/python3
+# Copyright (C) 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+import logging.handlers
+import os
+import sys
+
+
+import apifw
+import slog
+import yaml
+
+
+import ick2
+
+
+transactions = slog.Counter()
+
+
+def counter():
+    return 'HTTP transaction {}'.format(transactions.increment())
+
+
+def dict_logger(log, stack_info=None):
+    ick2.log.log(exc_info=stack_info, **log)
+
+
+default_config = {
+    'token-public-key': None,
+    'token-audience': None,
+    'token-issuer': None,
+    'log': [],
+    'blobdir': None,
+}
+
+
+def load_config(filename, defconf):
+    conf = yaml.safe_load(open(filename, 'r'))
+    actual_config = dict(defconf)
+    actual_config.update(conf)
+    return actual_config
+
+
+def check_config(config, musthave):
+    for key in config:
+        if key not in musthave:
+            raise Exception('Config %s is not known' % key)
+    for key in musthave:
+        if config.get(key) is None:
+            raise Exception('Config %s must not be None' % key)
+
+
+def main():
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    handler = logging.StreamHandler(sys.stderr)
+    logger.addHandler(handler)
+    logging.info('Starting blob service main program')
+
+    try:
+
+        config_filename = os.environ.get('BLOB_SERVICE_CONFIG')
+        logging.info('config filename %r', config_filename)
+        if not config_filename:
+            logging.error('no BLOB_SERVICE_CONFIG in environment')
+            raise Exception('No BLOB_SERVICE_CONFIG defined in environment')
+
+        logging.info('reading config from %r', config_filename)
+        config = load_config(config_filename, default_config)
+
+        logging.info('config is %r', config)
+        ick2.setup_logging(config)
+        check_config(config, default_config)
+
+        ick2.log.log('info', msg_text='Blob service starts', config=config)
+
+        api = ick2.BlobAPI()
+        ick2.log.log('info', msg_text='created BlobAPI')
+
+        api.set_blob_directory(config['blobdir'])
+        ick2.log.log(
+            'info', msg_text='called BlobAPI.set_blob_directory')
+
+        application = apifw.create_bottle_application(
+            api, counter, dict_logger, config)
+        ick2.log.log('info', msg_text='called apifw.create_bottle_application')
+
+        return application
+    except SystemExit:
+        raise
+    except BaseException as e:
+        logging.error(str(e))
+        ick2.log.log(
+            'error', msg_text='Uncaught  exception', exception=str(e),
+            exc_info=True)
+        sys.exit(1)
+
+
+app = main()
+
+# If we are running this program directly with Python, and not via
+# gunicorn, we can use the Bottle built-in debug server, which can
+# make some things easier to debug.
+
+if __name__ == '__main__':
+    print('running in debug mode')
+    app.run(host='127.0.0.1', port=12765)
diff --git a/ick2/__init__.py b/ick2/__init__.py
index 9b4d9e3..1fffe31 100644
--- a/ick2/__init__.py
+++ b/ick2/__init__.py
@@ -41,3 +41,5 @@ from .workerapi import WorkerAPI
 from .controllerapi import (
     ControllerAPI,
 )
+from .blobapi import BlobAPI
+from .blob_store import BlobStore
diff --git a/ick2/blob_store.py b/ick2/blob_store.py
new file mode 100644
index 0000000..7d02cbf
--- /dev/null
+++ b/ick2/blob_store.py
@@ -0,0 +1,51 @@
+# Copyright (C) 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+
+
+import ick2
+
+
+class BlobStore:
+
+    def __init__(self):
+        self._blobdir = None
+
+    def get_blob_directory(self):
+        return self._blobdir
+
+    def set_blob_directory(self, dirname):
+        self._blobdir = dirname
+        if not os.path.exists(self._blobdir):
+            os.makedirs(self._blobdir)
+
+    def store_blob(self, name, blob):
+        filename = self._blob_filename(name)
+        ick2.log.log(
+            'trace', msg_text='store_blob', name=name, filename=filename)
+        with open(filename, 'wb') as f:
+            f.write(blob)
+
+    def get_blob(self, name):
+        filename = self._blob_filename(name)
+        ick2.log.log(
+            'trace', msg_text='get_blob', name=name, filename=filename)
+        with open(filename, 'rb') as f:
+            return f.read()
+
+    def _blob_filename(self, name):
+        return os.path.join(self._blobdir, name)
diff --git a/ick2/blobapi.py b/ick2/blobapi.py
new file mode 100644
index 0000000..1772a27
--- /dev/null
+++ b/ick2/blobapi.py
@@ -0,0 +1,62 @@
+# Copyright 2017  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import ick2
+
+
+class BlobAPI:
+
+    def __init__(self):
+        self._blobs = ick2.BlobStore()
+
+    def get_blob_directory(self):
+        return self._blobs.get_blob_directory()
+
+    def set_blob_directory(self, dirname):
+        self._blobs.set_blob_directory(dirname)
+
+    def find_missing_route(self, missing_path):
+        path = '/blobs/<blob_id>'
+        return [
+            {
+                'method': 'PUT',
+                'path': path,
+                'callback': self.put_blob,
+            },
+            {
+                'method': 'GET',
+                'path': path,
+                'callback': self.get_blob,
+            },
+        ]
+
+    def put_blob(self, content_type, body, blob_id, **kwargs):
+        ick2.log.log(
+            'info', msg_text='blob uploaded', blob_id=blob_id, kwargs=kwargs)
+        self._blobs.store_blob(blob_id, body)
+        return ick2.OK('')
+
+    def get_blob(self, content_type, body, blob_id, **kwargs):
+        ick2.log.log(
+            'info', msg_text='blob requested', blob_id=blob_id, kwargs=kwargs)
+        try:
+            blob = self._blobs.get_blob(blob_id)
+        except EnvironmentError as e:
+            return ick2.not_found(str(e))
+        headers = {
+            'Content-Type': 'application/octet-stream',
+        }
+        return ick2.OK(blob, headers=headers)
diff --git a/run-blob-service-debug b/run-blob-service-debug
new file mode 100755
index 0000000..77bade4
--- /dev/null
+++ b/run-blob-service-debug
@@ -0,0 +1,35 @@
+#!/bin/sh
+# Copyright 2017 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eu
+
+scopes="
+uapi_blobs_id_put
+uapi_blobs_id_get
+"
+
+./generate-rsa-key t.key
+./create-token < t.key "$scopes" > t.token
+cat <<EOF > t.yaml
+log:
+- filename: t.log
+token-issuer: localhost
+token-audience: localhost
+token-public-key: $(cat t.key.pub)
+blobdir: t.blobs
+EOF
+
+BLOB_SERVICE_CONFIG=t.yaml python3 blob_service.py
diff --git a/without-tests b/without-tests
index 6b58af3..f7425bf 100644
--- a/without-tests
+++ b/without-tests
@@ -1,5 +1,7 @@
 ick2/__init__.py
 ick2/apibase.py
+ick2/blobapi.py
+ick2/blob_store.py
 ick2/buildsapi.py
 ick2/exceptions.py
 ick2/logapi.py
diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn
index 6f9d423..afddb3b 100644
--- a/yarns/900-implements.yarn
+++ b/yarns/900-implements.yarn
@@ -45,11 +45,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     path = get_next_match()
     token = get_token(user)
     url = vars['bsurl']
-    status, content_type, headers, body = get(url + path, token)
+    status, content_type, headers, body = get_blob(url + path, token)
     vars['status_code'] = status
     vars['content_type'] = content_type
     vars['headers'] = headers
-    vars['body'] = body
+    vars['body'] = body.encode('hex')
 
     IMPLEMENTS WHEN (\S+) makes request GET (\S+) with an invalid token
     user = get_next_match()
@@ -88,7 +88,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     vars['status_code'] = status
     vars['content_type'] = content_type
     vars['headers'] = headers
-    vars['body'] = body
+    vars['body'] = body.encode('hex')
 
     IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with a valid token and body (.+)
     user = get_next_match()
@@ -162,7 +162,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
     IMPLEMENTS THEN body is the same as the blob (\S+)
     filename = get_next_match()
     blob = cat(filename)
-    body = vars['body']
+    body = vars['body'].decode('hex')
     assertEqual(body, blob)
 
     IMPLEMENTS THEN version in body matches version from setup.py
diff --git a/yarns/lib.py b/yarns/lib.py
index 00db015..6575e6d 100644
--- a/yarns/lib.py
+++ b/yarns/lib.py
@@ -104,6 +104,14 @@ def get(url, token):
     return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text
 
 
+def get_blob(url, token):
+    headers = {
+        'Authorization': 'Bearer {}'.format(token),
+    }
+    r = requests.get(url, headers=headers, verify=False)
+    return r.status_code, r.headers['Content-Type'], dict(r.headers), r.content
+
+
 def post(url, body_text, token):
     headers = {
         'Authorization': 'Bearer {}'.format(token),