From 4e9b1ddfae711ce38d743a3faccf49447ab10f75 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sun, 3 Dec 2017 19:07:24 +0200 Subject: Add: blob service --- blob_service.py | 122 ++++++++++++++++++++++++++++++++++++++++++++++ ick2/__init__.py | 2 + ick2/blob_store.py | 51 +++++++++++++++++++ ick2/blobapi.py | 62 +++++++++++++++++++++++ run-blob-service-debug | 35 +++++++++++++ without-tests | 2 + yarns/900-implements.yarn | 8 +-- yarns/lib.py | 8 +++ 8 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 blob_service.py create mode 100644 ick2/blob_store.py create mode 100644 ick2/blobapi.py create mode 100755 run-blob-service-debug diff --git a/blob_service.py b/blob_service.py new file mode 100644 index 0000000..44141f6 --- /dev/null +++ b/blob_service.py @@ -0,0 +1,122 @@ +#!/usr/bin/python3 +# Copyright (C) 2017 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import logging +import logging.handlers +import os +import sys + + +import apifw +import slog +import yaml + + +import ick2 + + +transactions = slog.Counter() + + +def counter(): + return 'HTTP transaction {}'.format(transactions.increment()) + + +def dict_logger(log, stack_info=None): + ick2.log.log(exc_info=stack_info, **log) + + +default_config = { + 'token-public-key': None, + 'token-audience': None, + 'token-issuer': None, + 'log': [], + 'blobdir': None, +} + + +def load_config(filename, defconf): + conf = yaml.safe_load(open(filename, 'r')) + actual_config = dict(defconf) + actual_config.update(conf) + return actual_config + + +def check_config(config, musthave): + for key in config: + if key not in musthave: + raise Exception('Config %s is not known' % key) + for key in musthave: + if config.get(key) is None: + raise Exception('Config %s must not be None' % key) + + +def main(): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(sys.stderr) + logger.addHandler(handler) + logging.info('Starting blob service main program') + + try: + + config_filename = os.environ.get('BLOB_SERVICE_CONFIG') + logging.info('config filename %r', config_filename) + if not config_filename: + logging.error('no BLOB_SERVICE_CONFIG in environment') + raise Exception('No BLOB_SERVICE_CONFIG defined in environment') + + logging.info('reading config from %r', config_filename) + config = load_config(config_filename, default_config) + + logging.info('config is %r', config) + ick2.setup_logging(config) + check_config(config, default_config) + + ick2.log.log('info', msg_text='Blob service starts', config=config) + + api = ick2.BlobAPI() + ick2.log.log('info', msg_text='created BlobAPI') + + api.set_blob_directory(config['blobdir']) + ick2.log.log( + 'info', msg_text='called BlobAPI.set_blob_directory') + + application = apifw.create_bottle_application( + api, counter, dict_logger, config) + ick2.log.log('info', msg_text='called apifw.create_bottle_application') + + return application + except SystemExit: + raise + except BaseException as e: + logging.error(str(e)) + ick2.log.log( + 'error', msg_text='Uncaught exception', exception=str(e), + exc_info=True) + sys.exit(1) + + +app = main() + +# If we are running this program directly with Python, and not via +# gunicorn, we can use the Bottle built-in debug server, which can +# make some things easier to debug. + +if __name__ == '__main__': + print('running in debug mode') + app.run(host='127.0.0.1', port=12765) diff --git a/ick2/__init__.py b/ick2/__init__.py index 9b4d9e3..1fffe31 100644 --- a/ick2/__init__.py +++ b/ick2/__init__.py @@ -41,3 +41,5 @@ from .workerapi import WorkerAPI from .controllerapi import ( ControllerAPI, ) +from .blobapi import BlobAPI +from .blob_store import BlobStore diff --git a/ick2/blob_store.py b/ick2/blob_store.py new file mode 100644 index 0000000..7d02cbf --- /dev/null +++ b/ick2/blob_store.py @@ -0,0 +1,51 @@ +# Copyright (C) 2017 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import os + + +import ick2 + + +class BlobStore: + + def __init__(self): + self._blobdir = None + + def get_blob_directory(self): + return self._blobdir + + def set_blob_directory(self, dirname): + self._blobdir = dirname + if not os.path.exists(self._blobdir): + os.makedirs(self._blobdir) + + def store_blob(self, name, blob): + filename = self._blob_filename(name) + ick2.log.log( + 'trace', msg_text='store_blob', name=name, filename=filename) + with open(filename, 'wb') as f: + f.write(blob) + + def get_blob(self, name): + filename = self._blob_filename(name) + ick2.log.log( + 'trace', msg_text='get_blob', name=name, filename=filename) + with open(filename, 'rb') as f: + return f.read() + + def _blob_filename(self, name): + return os.path.join(self._blobdir, name) diff --git a/ick2/blobapi.py b/ick2/blobapi.py new file mode 100644 index 0000000..1772a27 --- /dev/null +++ b/ick2/blobapi.py @@ -0,0 +1,62 @@ +# Copyright 2017 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import ick2 + + +class BlobAPI: + + def __init__(self): + self._blobs = ick2.BlobStore() + + def get_blob_directory(self): + return self._blobs.get_blob_directory() + + def set_blob_directory(self, dirname): + self._blobs.set_blob_directory(dirname) + + def find_missing_route(self, missing_path): + path = '/blobs/' + return [ + { + 'method': 'PUT', + 'path': path, + 'callback': self.put_blob, + }, + { + 'method': 'GET', + 'path': path, + 'callback': self.get_blob, + }, + ] + + def put_blob(self, content_type, body, blob_id, **kwargs): + ick2.log.log( + 'info', msg_text='blob uploaded', blob_id=blob_id, kwargs=kwargs) + self._blobs.store_blob(blob_id, body) + return ick2.OK('') + + def get_blob(self, content_type, body, blob_id, **kwargs): + ick2.log.log( + 'info', msg_text='blob requested', blob_id=blob_id, kwargs=kwargs) + try: + blob = self._blobs.get_blob(blob_id) + except EnvironmentError as e: + return ick2.not_found(str(e)) + headers = { + 'Content-Type': 'application/octet-stream', + } + return ick2.OK(blob, headers=headers) diff --git a/run-blob-service-debug b/run-blob-service-debug new file mode 100755 index 0000000..77bade4 --- /dev/null +++ b/run-blob-service-debug @@ -0,0 +1,35 @@ +#!/bin/sh +# Copyright 2017 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +set -eu + +scopes=" +uapi_blobs_id_put +uapi_blobs_id_get +" + +./generate-rsa-key t.key +./create-token < t.key "$scopes" > t.token +cat < t.yaml +log: +- filename: t.log +token-issuer: localhost +token-audience: localhost +token-public-key: $(cat t.key.pub) +blobdir: t.blobs +EOF + +BLOB_SERVICE_CONFIG=t.yaml python3 blob_service.py diff --git a/without-tests b/without-tests index 6b58af3..f7425bf 100644 --- a/without-tests +++ b/without-tests @@ -1,5 +1,7 @@ ick2/__init__.py ick2/apibase.py +ick2/blobapi.py +ick2/blob_store.py ick2/buildsapi.py ick2/exceptions.py ick2/logapi.py diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn index 6f9d423..afddb3b 100644 --- a/yarns/900-implements.yarn +++ b/yarns/900-implements.yarn @@ -45,11 +45,11 @@ along with this program. If not, see . path = get_next_match() token = get_token(user) url = vars['bsurl'] - status, content_type, headers, body = get(url + path, token) + status, content_type, headers, body = get_blob(url + path, token) vars['status_code'] = status vars['content_type'] = content_type vars['headers'] = headers - vars['body'] = body + vars['body'] = body.encode('hex') IMPLEMENTS WHEN (\S+) makes request GET (\S+) with an invalid token user = get_next_match() @@ -88,7 +88,7 @@ along with this program. If not, see . vars['status_code'] = status vars['content_type'] = content_type vars['headers'] = headers - vars['body'] = body + vars['body'] = body.encode('hex') IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with a valid token and body (.+) user = get_next_match() @@ -162,7 +162,7 @@ along with this program. If not, see . IMPLEMENTS THEN body is the same as the blob (\S+) filename = get_next_match() blob = cat(filename) - body = vars['body'] + body = vars['body'].decode('hex') assertEqual(body, blob) IMPLEMENTS THEN version in body matches version from setup.py diff --git a/yarns/lib.py b/yarns/lib.py index 00db015..6575e6d 100644 --- a/yarns/lib.py +++ b/yarns/lib.py @@ -104,6 +104,14 @@ def get(url, token): return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text +def get_blob(url, token): + headers = { + 'Authorization': 'Bearer {}'.format(token), + } + r = requests.get(url, headers=headers, verify=False) + return r.status_code, r.headers['Content-Type'], dict(r.headers), r.content + + def post(url, body_text, token): headers = { 'Authorization': 'Bearer {}'.format(token), -- cgit v1.2.1