From 78d9c80226c52bcd70e27e7ee2a3a7754f644b53 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 24 Mar 2018 14:02:20 +0200 Subject: Change: rename blob service to artifact store --- NEWS | 1 + artifact_store.py | 123 ++++++++++++++++++++++++++++++++++++++++++ artifact_store.service | 29 ++++++++++ blob_service.py | 122 ----------------------------------------- debian/ick2.postinst | 6 +-- ick-blob-service.service | 29 ---------- ick2/controllerapi.py | 6 +-- ick2/versionapi.py | 8 +-- ick2/versionapi_tests.py | 4 +- ick_controller.py | 6 +-- icktool | 6 +-- run-artifact-store-debug | 35 ++++++++++++ run-blob-service-debug | 35 ------------ setup.py | 6 +-- start_artifact_store | 24 +++++++++ start_blob_service | 24 --------- worker_manager | 8 +-- yarns/100-projects.yarn | 2 +- yarns/150-pipelines.yarn | 2 +- yarns/200-version.yarn | 4 +- yarns/300-workers.yarn | 2 +- yarns/400-build.yarn | 6 +-- yarns/500-build-fail.yarn | 2 +- yarns/600-unauthz.yarn | 2 +- yarns/700-artifact-store.yarn | 52 ++++++++++++++++++ yarns/700-blob-service.yarn | 52 ------------------ yarns/900-implements.yarn | 4 +- yarns/900-local.yarn | 32 +++++------ 28 files changed, 317 insertions(+), 315 deletions(-) create mode 100644 artifact_store.py create mode 100644 artifact_store.service delete mode 100644 blob_service.py delete mode 100644 ick-blob-service.service create mode 100755 run-artifact-store-debug delete mode 100755 run-blob-service-debug create mode 100644 start_artifact_store delete mode 100644 start_blob_service create mode 100644 yarns/700-artifact-store.yarn delete mode 100644 yarns/700-blob-service.yarn diff --git a/NEWS b/NEWS index 21a5630..0fe199b 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,7 @@ along with this program. If not, see . Version 0.28+git, not yet released ---------------------------------- +* "Blob service" has been renamed to "artifact store". Version 0.28, released 2018-02-24 ---------------------------------- diff --git a/artifact_store.py b/artifact_store.py new file mode 100644 index 0000000..3a19079 --- /dev/null +++ b/artifact_store.py @@ -0,0 +1,123 @@ +#!/usr/bin/python3 +# Copyright (C) 2018 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import logging +import logging.handlers +import os +import sys + + +import apifw +import slog +import yaml + + +import ick2 + + +transactions = slog.Counter() + + +def counter(): + return 'HTTP transaction {}'.format(transactions.increment()) + + +def dict_logger(log, stack_info=None): + ick2.log.log(exc_info=stack_info, **log) + + +default_config = { + 'token-public-key': None, + 'token-audience': None, + 'token-issuer': None, + 'log': [], + 'blobdir': None, +} + + +def load_config(filename, defconf): + conf = yaml.safe_load(open(filename, 'r')) + actual_config = dict(defconf) + actual_config.update(conf) + return actual_config + + +def check_config(config, musthave): + for key in config: + if key not in musthave: + raise Exception('Config %s is not known' % key) + for key in musthave: + if config.get(key) is None: + raise Exception('Config %s must not be None' % key) + + +def main(): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(sys.stderr) + logger.addHandler(handler) + logging.info('Starting artifact store main program') + + try: + + name = 'ARTIFACT_STORE_CONFIG' + config_filename = os.environ.get(name) + logging.info('config filename %r', config_filename) + if not config_filename: + msg = 'No %s defined in environment' % name + logging.error(msg) + raise Exception(msg) + + logging.info('reading config from %r', config_filename) + config = load_config(config_filename, default_config) + + logging.info('config is %r', config) + ick2.setup_logging(config) + check_config(config, default_config) + + ick2.log.log('info', msg_text='Artifact store starts', config=config) + + api = ick2.BlobAPI() + ick2.log.log('info', msg_text='created BlobAPI') + + api.set_blob_directory(config['blobdir']) + ick2.log.log('info', msg_text='called BlobAPI.set_blob_directory') + + application = apifw.create_bottle_application( + api, counter, dict_logger, config) + ick2.log.log('info', msg_text='called apifw.create_bottle_application') + + return application + except SystemExit: + raise + except BaseException as e: + logging.error(str(e)) + ick2.log.log( + 'error', msg_text='Uncaught exception', exception=str(e), + exc_info=True) + sys.exit(1) + + +app = main() + +# If we are running this program directly with Python, and not via +# gunicorn, we can use the Bottle built-in debug server, which can +# make some things easier to debug. + +if __name__ == '__main__': + print('running in debug mode') + app.run(host='127.0.0.1', port=12765) diff --git a/artifact_store.service b/artifact_store.service new file mode 100644 index 0000000..a63d034 --- /dev/null +++ b/artifact_store.service @@ -0,0 +1,29 @@ +# Copyright 2017-2018 Lars Wirzenius +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +[Unit] +Description=ick artifact store +After=network.target +ConditionPathExists=/etc/ick/artifact_store.yaml + +[Service] +Type=simple +User=_ickas +Group=_ickas +WorkingDirectory=/var/lib/ick/blobs +ExecStart=/usr/bin/start_artifact_store +KillSignal=QUIT + +[Install] +WantedBy=multi-user.target diff --git a/blob_service.py b/blob_service.py deleted file mode 100644 index 44141f6..0000000 --- a/blob_service.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/python3 -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - - -import logging -import logging.handlers -import os -import sys - - -import apifw -import slog -import yaml - - -import ick2 - - -transactions = slog.Counter() - - -def counter(): - return 'HTTP transaction {}'.format(transactions.increment()) - - -def dict_logger(log, stack_info=None): - ick2.log.log(exc_info=stack_info, **log) - - -default_config = { - 'token-public-key': None, - 'token-audience': None, - 'token-issuer': None, - 'log': [], - 'blobdir': None, -} - - -def load_config(filename, defconf): - conf = yaml.safe_load(open(filename, 'r')) - actual_config = dict(defconf) - actual_config.update(conf) - return actual_config - - -def check_config(config, musthave): - for key in config: - if key not in musthave: - raise Exception('Config %s is not known' % key) - for key in musthave: - if config.get(key) is None: - raise Exception('Config %s must not be None' % key) - - -def main(): - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - handler = logging.StreamHandler(sys.stderr) - logger.addHandler(handler) - logging.info('Starting blob service main program') - - try: - - config_filename = os.environ.get('BLOB_SERVICE_CONFIG') - logging.info('config filename %r', config_filename) - if not config_filename: - logging.error('no BLOB_SERVICE_CONFIG in environment') - raise Exception('No BLOB_SERVICE_CONFIG defined in environment') - - logging.info('reading config from %r', config_filename) - config = load_config(config_filename, default_config) - - logging.info('config is %r', config) - ick2.setup_logging(config) - check_config(config, default_config) - - ick2.log.log('info', msg_text='Blob service starts', config=config) - - api = ick2.BlobAPI() - ick2.log.log('info', msg_text='created BlobAPI') - - api.set_blob_directory(config['blobdir']) - ick2.log.log( - 'info', msg_text='called BlobAPI.set_blob_directory') - - application = apifw.create_bottle_application( - api, counter, dict_logger, config) - ick2.log.log('info', msg_text='called apifw.create_bottle_application') - - return application - except SystemExit: - raise - except BaseException as e: - logging.error(str(e)) - ick2.log.log( - 'error', msg_text='Uncaught exception', exception=str(e), - exc_info=True) - sys.exit(1) - - -app = main() - -# If we are running this program directly with Python, and not via -# gunicorn, we can use the Bottle built-in debug server, which can -# make some things easier to debug. - -if __name__ == '__main__': - print('running in debug mode') - app.run(host='127.0.0.1', port=12765) diff --git a/debian/ick2.postinst b/debian/ick2.postinst index 822006b..35578ed 100644 --- a/debian/ick2.postinst +++ b/debian/ick2.postinst @@ -36,7 +36,7 @@ create_user() # Create a group and user for ick users create_user _ick /var/lib/ick /var/log/ick create_user _ickwm /var/lib/ick/wm /var/log/ickwm -create_user _ickbs /var/lib/ick/blobs /var/log/ickbs +create_user _ickas /var/lib/ick/blobs /var/log/ickas # Create controller config dir. install -d -m 0755 -o root -g root /etc/ick @@ -50,7 +50,7 @@ install -d -m 0755 -o _ickwm -g _ickwm /var/lib/ick/workspace # Create worker-manager systree install -d -m 0755 -o _ickwm -g _ickwm /var/lib/ick/systree -# Create blob service storage -install -d -m 0755 -o _ickbs -g _ickbs /var/lib/ick/blobs +# Create artifact store storage +install -d -m 0755 -o _ickas -g _ickas /var/lib/ick/blobs #DEBHELPER# diff --git a/ick-blob-service.service b/ick-blob-service.service deleted file mode 100644 index 2fbb2f2..0000000 --- a/ick-blob-service.service +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2017-2018 Lars Wirzenius -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -[Unit] -Description=ick blob service -After=network.target -ConditionPathExists=/etc/ick/blob-service.yaml - -[Service] -Type=simple -User=_ickbs -Group=_ickbs -WorkingDirectory=/var/lib/ick/blobs -ExecStart=/usr/bin/start_blob_service -KillSignal=QUIT - -[Install] -WantedBy=multi-user.target diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py index 074fa2e..f7c71f6 100644 --- a/ick2/controllerapi.py +++ b/ick2/controllerapi.py @@ -22,13 +22,13 @@ class ControllerAPI: self._state = state self._apis = {} - def set_blob_service_url(self, url): # pragma: no cover + def set_artifact_store_url(self, url): # pragma: no cover self.find_missing_route('/version') api = self._apis.get('/version') if api: - api.set_blob_service_url(url) + api.set_artifact_store_url(url) ick2.log.log( - 'info', msg_text='Set blob service url', url=url, + 'info', msg_text='Set artifact store url', url=url, version=api.get_version()) def find_missing_route(self, missing_path): # pragma: no cover diff --git a/ick2/versionapi.py b/ick2/versionapi.py index 40d1dda..cc7e169 100644 --- a/ick2/versionapi.py +++ b/ick2/versionapi.py @@ -20,10 +20,10 @@ class VersionAPI(ick2.APIbase): def __init__(self, state): super().__init__(state) - self._blob_service_url = None + self._artifact_store_url = None - def set_blob_service_url(self, url): - self._blob_service_url = url + def set_artifact_store_url(self, url): + self._artifact_store_url = url def get_routes(self, path): # pragma: no cover return [ @@ -37,7 +37,7 @@ class VersionAPI(ick2.APIbase): def get_version(self, **kwargs): return { 'version': ick2.__version__, - 'blob_service': self._blob_service_url, + 'artifact_store': self._artifact_store_url, } def create(self, body, **kwargs): # pragma: no cover diff --git a/ick2/versionapi_tests.py b/ick2/versionapi_tests.py index 143e1cc..b8d59d5 100644 --- a/ick2/versionapi_tests.py +++ b/ick2/versionapi_tests.py @@ -24,12 +24,12 @@ class VersionAPITests(unittest.TestCase): def test_returns_version_correcly(self): bloburl = 'https://blobs.example.com' api = ick2.VersionAPI(None) - api.set_blob_service_url(bloburl) + api.set_artifact_store_url(bloburl) response = api.get_version() self.assertEqual( response, { 'version': ick2.__version__, - 'blob_service': bloburl, + 'artifact_store': bloburl, } ) diff --git a/ick_controller.py b/ick_controller.py index a176351..edfebb4 100644 --- a/ick_controller.py +++ b/ick_controller.py @@ -46,7 +46,7 @@ default_config = { 'token-issuer': None, 'log': [], 'statedir': None, - 'blob-service': None, + 'artifact-store': None, } @@ -94,10 +94,10 @@ def main(): state.set_state_directory(config['statedir']) api = ick2.ControllerAPI(state) - api.set_blob_service_url(config['blob-service']) + api.set_artifact_store_url(config['artifact-store']) ick2.log.log( 'info', msg_text='created ControllerAPI', - blob_service=config['blob-service']) + artifact_store=config['artifact-store']) application = apifw.create_bottle_application( api, counter, dict_logger, config) diff --git a/icktool b/icktool index 11229bd..625d9b4 100755 --- a/icktool +++ b/icktool @@ -364,7 +364,7 @@ class Icktool(cliapp.Application): token = self.settings['token'] or self._new_token() blob_api = BlobAPI() blob_api.set_token(token) - blob_api.set_url(api.get_blob_service_url()) + blob_api.set_url(api.get_artifact_store_url()) blob_api.set_verify(self.settings['verify-tls']) return blob_api @@ -429,10 +429,10 @@ class API: if code == 200: return json.loads(text) - def get_blob_service_url(self): + def get_artifact_store_url(self): version = self.get_version() if version: - return version.get('blob_service') + return version.get('artifact_store') def get(self, path): assert self._url is not None diff --git a/run-artifact-store-debug b/run-artifact-store-debug new file mode 100755 index 0000000..6617abc --- /dev/null +++ b/run-artifact-store-debug @@ -0,0 +1,35 @@ +#!/bin/sh +# Copyright 2017-2018 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +set -eu + +scopes=" +uapi_blobs_id_put +uapi_blobs_id_get +" + +./generate-rsa-key t.key +./create-token < t.key "$scopes" > t.token +cat < t.yaml +log: +- filename: t.log +token-issuer: localhost +token-audience: localhost +token-public-key: $(cat t.key.pub) +blobdir: t.blobs +EOF + +ARTIFACT_STORE_CONFIG=t.yaml python3 artifact_store.py diff --git a/run-blob-service-debug b/run-blob-service-debug deleted file mode 100755 index 77bade4..0000000 --- a/run-blob-service-debug +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# Copyright 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -set -eu - -scopes=" -uapi_blobs_id_put -uapi_blobs_id_get -" - -./generate-rsa-key t.key -./create-token < t.key "$scopes" > t.token -cat < t.yaml -log: -- filename: t.log -token-issuer: localhost -token-audience: localhost -token-public-key: $(cat t.key.pub) -blobdir: t.blobs -EOF - -BLOB_SERVICE_CONFIG=t.yaml python3 blob_service.py diff --git a/setup.py b/setup.py index 8cdc0c7..70b684a 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright (C) 2017 Lars Wirzenius +# Copyright (C) 2017-2018 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -48,13 +48,13 @@ setup( url='http://liw.fi/ick/', py_modules=[ 'ick_controller', - 'blob_service', + 'artifact_store', ], packages=['ick2'], scripts=[ 'create-token', 'start_ick', - 'start_blob_service', + 'start_artifact_store', 'icktool', 'worker_manager' ], diff --git a/start_artifact_store b/start_artifact_store new file mode 100644 index 0000000..3bc4a40 --- /dev/null +++ b/start_artifact_store @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright (C) 2017-2018 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +set -eux + +export ARTIFACT_STORE_CONFIG=/etc/ick/artifact_store.yaml +gunicorn3 \ + --bind 127.0.0.1:12766 \ + --log-file /var/log/ickas/gunicorn3.log \ + --log-level debug \ + artifact_store:app diff --git a/start_blob_service b/start_blob_service deleted file mode 100644 index b4a929f..0000000 --- a/start_blob_service +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -# Copyright (C) 2017-2018 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -set -eux - -export BLOB_SERVICE_CONFIG=/etc/ick/blob-service.yaml -gunicorn3 \ - --bind 127.0.0.1:12766 \ - --log-file /var/log/ickbs/gunicorn3.log \ - --log-level debug \ - blob_service:app diff --git a/worker_manager b/worker_manager index ad51619..7530469 100755 --- a/worker_manager +++ b/worker_manager @@ -175,19 +175,19 @@ class ControllerAPI: def bloburl(self, blob_id): if self._blob_url is None: - self._blob_url = self.get_blob_service_url() + self._blob_url = self.get_artifact_store_url() if self._blob_url is not None: return '{}/blobs/{}'.format(self._blob_url, blob_id) - logging.error('Do not have blob service URL') + logging.error('Do not artifact store URL') return None - def get_blob_service_url(self): + def get_artifact_store_url(self): url = self.url('/version') headers = self.get_auth_headers() version = self._httpapi.get(url, headers) logging.info('Version: %r', version) if version: - return version.get('blob_service') + return version.get('artifact_store') class HttpApi: diff --git a/yarns/100-projects.yarn b/yarns/100-projects.yarn index 7c18952..b5a7f78 100644 --- a/yarns/100-projects.yarn +++ b/yarns/100-projects.yarn @@ -55,7 +55,7 @@ building them. We start by starting an instance of the controller. ... uapi_projects_id_put ... uapi_projects_id_delete AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND a running ick controller WHEN user makes request GET /projects diff --git a/yarns/150-pipelines.yarn b/yarns/150-pipelines.yarn index feda6d0..15c4b05 100644 --- a/yarns/150-pipelines.yarn +++ b/yarns/150-pipelines.yarn @@ -63,7 +63,7 @@ running them. We start by starting an instance of the controller. ... uapi_pipelines_id_put ... uapi_pipelines_id_delete AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND a running ick controller WHEN user makes request GET /pipelines diff --git a/yarns/200-version.yarn b/yarns/200-version.yarn index 2f87ac7..d281a19 100644 --- a/yarns/200-version.yarn +++ b/yarns/200-version.yarn @@ -26,13 +26,13 @@ The Ick controller reports is version upon request. AND an access token for user with scopes ... uapi_version_get AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND a running ick controller WHEN user makes request GET /version THEN result has status code 200 AND version in body matches version from setup.py - AND blob service URL is https://blobs.example.com + AND artifact store URL is https://blobs.example.com FINALLY stop ick controller diff --git a/yarns/300-workers.yarn b/yarns/300-workers.yarn index 2cc7ea2..8b12f6f 100644 --- a/yarns/300-workers.yarn +++ b/yarns/300-workers.yarn @@ -60,7 +60,7 @@ controller API. It doesn't actually talk to the worker itself. ... uapi_workers_id_put ... uapi_workers_id_delete AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND a running ick controller WHEN user makes request GET /workers diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn index ef164d9..e2ba579 100644 --- a/yarns/400-build.yarn +++ b/yarns/400-build.yarn @@ -27,7 +27,7 @@ Set up the controller. GIVEN an RSA key pair for token signing AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND an access token for user with scopes ... uapi_pipelines_post ... uapi_projects_post @@ -640,7 +640,7 @@ Set up the controller. GIVEN an RSA key pair for token signing AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND an access token for user with scopes ... uapi_pipelines_post ... uapi_projects_post @@ -797,7 +797,7 @@ Set up the controller. GIVEN an RSA key pair for token signing AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND an access token for user with scopes ... uapi_pipelines_post ... uapi_projects_post diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn index dcc551d..84ec66d 100644 --- a/yarns/500-build-fail.yarn +++ b/yarns/500-build-fail.yarn @@ -28,7 +28,7 @@ Set up the controller. GIVEN an RSA key pair for token signing AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND an access token for user with scopes ... uapi_pipelines_post ... uapi_projects_post diff --git a/yarns/600-unauthz.yarn b/yarns/600-unauthz.yarn index 0a602af..c57f815 100644 --- a/yarns/600-unauthz.yarn +++ b/yarns/600-unauthz.yarn @@ -28,7 +28,7 @@ Set up the controller. GIVEN an RSA key pair for token signing AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as blob service + AND controller config uses https://blobs.example.com as artifact store AND an access token for user with scopes ... uapi_projects_post ... uapi_projects_id_pipelines_id_put diff --git a/yarns/700-artifact-store.yarn b/yarns/700-artifact-store.yarn new file mode 100644 index 0000000..2dcea2e --- /dev/null +++ b/yarns/700-artifact-store.yarn @@ -0,0 +1,52 @@ + + +# Artifact store + +This scenario tests the artifact store API to store and retrieve +blobs. At this stage the artifact store is the simplest possible; so +simple, in fact, it will certainly change in the future. + + SCENARIO artifact store + +Set up the artifact store. + + GIVEN an RSA key pair for token signing + AND artifact store config uses blobs at the blob directory + AND an access token for user with scopes + ... uapi_blobs_id_put + ... uapi_blobs_id_get + AND a running artifact store + +Try to get a non-existent blob. It should result in an error. + + WHEN user retrieves /blobs/cake from artifact store + THEN result has status code 404 + +Create and store a blob, retrieve it and verify we get it back intack. + + WHEN user creates a blob named cake with random data + AND user sends blob cake to artifact store as /blobs/cake + THEN result has status code 200 + + WHEN user retrieves /blobs/cake from artifact store + THEN result has status code 200 + AND body is the same as the blob cake + + FINALLY stop artifact store diff --git a/yarns/700-blob-service.yarn b/yarns/700-blob-service.yarn deleted file mode 100644 index 1fd0c06..0000000 --- a/yarns/700-blob-service.yarn +++ /dev/null @@ -1,52 +0,0 @@ - - -# Blob service - -This scenario tests the blob service API to store and retrieve blobs. -At this stage the blob service is the simplest possible; so simple, in -fact, it will certainly change in the future. - - SCENARIO blob service - -Set up the blob service. - - GIVEN an RSA key pair for token signing - AND blob service config uses blobs at the blob directory - AND an access token for user with scopes - ... uapi_blobs_id_put - ... uapi_blobs_id_get - AND a running blob service - -Try to get a non-existent blob. It should result in an error. - - WHEN user retrieves /blobs/cake from blob service - THEN result has status code 404 - -Create and store a blob, retrieve it and verify we get it back intack. - - WHEN user creates a blob named cake with random data - AND user sends blob cake to blob service as /blobs/cake - THEN result has status code 200 - - WHEN user retrieves /blobs/cake from blob service - THEN result has status code 200 - AND body is the same as the blob cake - - FINALLY stop blob service diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn index c6be689..8a7dff1 100644 --- a/yarns/900-implements.yarn +++ b/yarns/900-implements.yarn @@ -36,7 +36,7 @@ along with this program. If not, see . url = vars['url'] http(vars, get, url + path, token=token) - IMPLEMENTS WHEN (\S+) retrieves (\S+) from blob service + IMPLEMENTS WHEN (\S+) retrieves (\S+) from artifact store user = get_next_match() path = get_next_match() token = get_token(user) @@ -74,7 +74,7 @@ along with this program. If not, see . url = vars['url'] http(vars, put, url + path, body=body, token=token) - IMPLEMENTS WHEN (\S+) sends blob (\S+) to blob service as (\S+) + IMPLEMENTS WHEN (\S+) sends blob (\S+) to artifact store as (\S+) user = get_next_match() filename = get_next_match() path = get_next_match() diff --git a/yarns/900-local.yarn b/yarns/900-local.yarn index 5efad69..4e9dbe1 100644 --- a/yarns/900-local.yarn +++ b/yarns/900-local.yarn @@ -46,8 +46,8 @@ along with this program. If not, see . IMPLEMENTS GIVEN controller config uses (\S+) at the state directory vars['statedir'] = get_next_match() - IMPLEMENTS GIVEN controller config uses (\S+) as blob service - vars['blob_service'] = get_next_match() + IMPLEMENTS GIVEN controller config uses (\S+) as artifact store + vars['artifact_store'] = get_next_match() ## Start and stop the controller @@ -67,7 +67,7 @@ along with this program. If not, see . }, ], 'statedir': vars['statedir'], - 'blob-service': vars['blob_service'], + 'artifact-store': vars['artifact_store'], } env = dict(os.environ) env['ICK_CONTROLLER_CONFIG'] = 'ick_controller.yaml' @@ -109,22 +109,22 @@ along with this program. If not, see . ## Check version result - IMPLEMENTS THEN blob service URL is (\S+) + IMPLEMENTS THEN artifact store URL is (\S+) expected = get_next_match() body = vars['body'] obj = json.loads(body) - actual = obj['blob_service'] + actual = obj['artifact_store'] assertEqual(actual, expected) -## Start and stop blob service +## Start and stop artifact store - IMPLEMENTS GIVEN blob service config uses (\S+) at the blob directory + IMPLEMENTS GIVEN artifact store config uses (\S+) at the blob directory vars['blobdir'] = get_next_match() - IMPLEMENTS GIVEN a running blob service + IMPLEMENTS GIVEN a running artifact store import os, time, cliapp, yaml - vars['blob_service.log'] = 'blob_service.log' - vars['gunicorn3_bs.log'] = 'gunicorn3_bs.log' + vars['artifact_store.log'] = 'artifact_store.log' + vars['gunicorn3_as.log'] = 'gunicorn3_as.log' vars['bsport'] = random_free_port() vars['bsurl'] = 'http://127.0.0.1:{}'.format(vars['bsport']) config = { @@ -133,27 +133,27 @@ along with this program. If not, see . 'token-public-key': cat('token.key.pub'), 'log': [ { - 'filename': vars['blob_service.log'], + 'filename': vars['artifact_store.log'], }, ], 'blobdir': vars['blobdir'], } env = dict(os.environ) - env['BLOB_SERVICE_CONFIG'] = 'blob_service.yaml' - yaml.safe_dump(config, open('blob_service.yaml', 'w')) + env['ARTIFACT_STORE_CONFIG'] = 'artifact_store.yaml' + yaml.safe_dump(config, open('artifact_store.yaml', 'w')) argv = [ 'gunicorn3', '--daemon', '--bind', '127.0.0.1:{}'.format(vars['bsport']), - '--log-file', vars['gunicorn3_bs.log'], + '--log-file', vars['gunicorn3_as.log'], '--log-level', 'debug', '-p', 'bspid', - 'blob_service:app', + 'artifact_store:app', ] cliapp.runcmd(argv, env=env) vars['bspid'] = int(cat('bspid')) wait_for_port(vars['bsport']) - IMPLEMENTS FINALLY stop blob service + IMPLEMENTS FINALLY stop artifact store import os, signal os.kill(vars['bspid'], signal.SIGTERM) -- cgit v1.2.1