summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--NEWS17
-rw-r--r--artifact_store.py4
-rwxr-xr-xcheck39
-rw-r--r--debian/changelog10
-rw-r--r--ick2/__init__.py14
-rw-r--r--ick2/apibase.py73
-rw-r--r--ick2/buildsapi.py4
-rw-r--r--ick2/controllerapi.py10
-rw-r--r--ick2/exceptions.py24
-rw-r--r--ick2/logapi.py6
-rw-r--r--ick2/persistent.py156
-rw-r--r--ick2/persistent_tests.py66
-rw-r--r--ick2/projectapi.py20
-rw-r--r--ick2/projectapi_tests.py23
-rw-r--r--ick2/resource.py58
-rw-r--r--ick2/store.py160
-rw-r--r--ick2/store_tests.py89
-rw-r--r--ick2/tokengetter.py70
-rw-r--r--ick2/trans.py113
-rw-r--r--ick2/version.py4
-rw-r--r--ick2/workapi.py27
-rw-r--r--ick2/workapi_tests.py28
-rw-r--r--ick2/workerapi.py3
-rw-r--r--ick_controller.py12
-rwxr-xr-xicktool6
-rw-r--r--notification_service.py4
-rwxr-xr-xstart_ick4
-rw-r--r--without-tests2
-rw-r--r--yarns/100-projects.yarn1
-rw-r--r--yarns/150-pipelines.yarn1
-rw-r--r--yarns/300-workers.yarn1
-rw-r--r--yarns/400-build.yarn3
-rw-r--r--yarns/500-build-fail.yarn1
-rw-r--r--yarns/600-unauthz.yarn1
-rw-r--r--yarns/700-artifact-store.yarn1
35 files changed, 687 insertions, 368 deletions
diff --git a/NEWS b/NEWS
index 170b67e..6681666 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,7 @@
NEWS for ick2, a CI server
=============================================================================
-Copyright 2017-2018 Lars Wirzenius
+Copyright 2017-2019 Lars Wirzenius
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
@@ -17,9 +17,22 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
-Version 0.53.2+git, not yet released
+Version 0.54+git, not yet released
+----------------------------------
+
+* Controller now uses Muck for persistent storage, rather than YAML
+ files on the local file system. This is a necessary step for
+ isolation different users from each other.
+
+* Fix icktool to request the Muck scopes as well.
+
+Version 0.54, released 2019-07-26
------------------------------------
+* A ton of changes. This NEWS entry isn't complete, sorry. I'm making
+ a release before merging in a large change to how the controller
+ stores persistent data.
+
* The worker manager now has an action to mirror several git
repositories at once: `action: git_mirror`. See
[the specification][].
diff --git a/artifact_store.py b/artifact_store.py
index ea1db6a..8dea22d 100644
--- a/artifact_store.py
+++ b/artifact_store.py
@@ -1,5 +1,5 @@
#!/usr/bin/python3
-# Copyright (C) 2018 Lars Wirzenius
+# Copyright (C) 2018-2019 Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -120,4 +120,4 @@ app = main()
if __name__ == '__main__':
print('running in debug mode')
- app.run(host='127.0.0.1', port=12766)
+ app.run(host='127.0.0.1', port=5555)
diff --git a/check b/check
index 1e98ee1..913f21d 100755
--- a/check
+++ b/check
@@ -31,11 +31,13 @@ title()
title Remote yarns?
+local=yes
yarns=no
if [ "$#" -gt 0 ]
then
case "$1" in
https://*)
+ local=no
yarns=yes
remote_url="$1"
shift 1
@@ -48,29 +50,32 @@ then
fi
-title Unit tests
-python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2
-
-if [ -e .git ]
+if [ "$local" = yes ]
then
- sources="$(git ls-files | grep -Fvxf copyright-exceptions)"
+ title Unit tests
+ python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2
- title Copyright statements
- copyright-statement-lint $sources
+ if [ -e .git ]
+ then
+ sources="$(git ls-files | grep -Fvxf copyright-exceptions)"
- title Copyright licences
- ./is-agpl3+ $sources
-fi
+ title Copyright statements
+ copyright-statement-lint $sources
-python_sources="ick_controller.py worker_manager ick2 icktool"
+ title Copyright licences
+ ./is-agpl3+ $sources
+ fi
-title pycodestyle
-pycodestyle ick2 $python_sources
+ python_sources="ick_controller.py worker_manager ick2 icktool"
-if command -v pylint3 > /dev/null
-then
- title pylint3
- pylint3 --rcfile pylint.conf $python_sources
+ title pycodestyle
+ pycodestyle ick2 $python_sources
+
+ if command -v pylint3 > /dev/null
+ then
+ title pylint3
+ pylint3 --rcfile pylint.conf $python_sources
+ fi
fi
if [ "$yarns" = yes ]
diff --git a/debian/changelog b/debian/changelog
index 5e1e63c..4257156 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,14 @@
-ick2 (0.53.2+git-1) UNRELEASED; urgency=medium
+ick2 (0.54+git-1) UNRELEASED; urgency=medium
* New upstream version.
- -- Lars Wirzenius <liw@liw.fi> Wed, 18 Jul 2018 20:03:56 +0300
+ -- Lars Wirzenius <liw@liw.fi> Fri, 26 Jul 2019 09:11:54 +0300
+
+ick2 (0.54-1) stretch; urgency=medium
+
+ * New upstream version.
+
+ -- Lars Wirzenius <liw@liw.fi> Fri, 26 Jul 2019 09:11:53 +0300
ick2 (0.53.2-1) stretch; urgency=medium
diff --git a/ick2/__init__.py b/ick2/__init__.py
index f489396..586ddb5 100644
--- a/ick2/__init__.py
+++ b/ick2/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -15,9 +15,11 @@
from .version import __version__, __version_info__
from .logging import setup_logging, log
-from .persistent import (
- FilePersistentState,
- NotFound,
+from .store import (
+ MemoryStore,
+ MuckStore,
+)
+from .resource import (
Resource,
resource_from_dict,
)
@@ -47,7 +49,10 @@ from .buildsm import (
)
from .exceptions import (
BadUpdate,
+ NotFound,
ExistsAlready,
+ Conflict,
+ StoreError,
IckException,
MethodNotAllowed,
ClientIdMissing,
@@ -84,6 +89,7 @@ from .client import (
AuthClient,
Reporter,
)
+from .tokengetter import TokenGetter
from .actionenvs import (
Runner,
ActionEnvironment,
diff --git a/ick2/apibase.py b/ick2/apibase.py
index c08f7cf..0537b6b 100644
--- a/ick2/apibase.py
+++ b/ick2/apibase.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -16,11 +16,20 @@
import ick2
+import bottle
+
+
class APIbase:
def __init__(self, state):
- assert state is None or isinstance(state, ick2.FilePersistentState)
+ assert (state is None or
+ isinstance(state, ick2.MemoryStore) or
+ isinstance(state, ick2.MuckStore))
self._trans = ick2.TransactionalState(state)
+ self._token_getter = None
+
+ def set_token_getter(self, getter): # pragma: no cover
+ self._token_getter = getter
def get_routes(self, path):
resource_path = '{}/<name:re:[^/+][^/]*?(/[^/+][^/]*?)*>'.format(path)
@@ -52,12 +61,26 @@ class APIbase:
},
]
+ # This is quite ugly. The apifw library we use as a wrapper around
+ # Bottle should be doing this. But due to stupid reasons, it's
+ # awkward to modify that library, for now, and anyway all of this
+ # is going to be rewritten in a better programming language
+ # eventually, so we take the past of least effort and increase
+ # technical debt.
+ def _get_token(self):
+ v = bottle.request.get_header('Authorization', '')
+ prefix = 'Bearer '
+ if v.startswith(prefix):
+ return v[len(prefix):]
+ return None
+
def GET(self, callback):
def wrapper(content_type, body, **kwargs):
ick2.log.log(
'trace', msg_text='GET called', kwargs=kwargs,
content_type=content_type, body=body)
try:
+ kwargs['token'] = self._get_token()
if 'raw_uri_path' in kwargs:
del kwargs['raw_uri_path']
body = callback(**kwargs)
@@ -82,6 +105,7 @@ class APIbase:
'trace', msg_text='POST called', kwargs=kwargs,
content_type=content_type, body=body)
try:
+ kwargs['token'] = self._get_token()
body = callback(body, **kwargs)
except ick2.ExistsAlready as e:
ick2.log.log('error', msg_text=str(e), kwargs=kwargs)
@@ -94,6 +118,7 @@ class APIbase:
ick2.log.log(
'trace', msg_text='PUT called', kwargs=kwargs,
content_type=content_type, body=body)
+ kwargs['token'] = self._get_token()
if 'raw_uri_path' in kwargs:
del kwargs['raw_uri_path']
try:
@@ -112,6 +137,7 @@ class APIbase:
'trace', msg_text='DELETE called', kwargs=kwargs,
content_type=content_type, body=body)
try:
+ kwargs['token'] = self._get_token()
if 'raw_uri_path' in kwargs:
del kwargs['raw_uri_path']
body = callback(**kwargs)
@@ -122,19 +148,19 @@ class APIbase:
return ick2.OK(body)
return wrapper
- def create(self, body, **kwargs):
+ def create(self, body, token=None, **kwargs):
raise NotImplementedError()
- def update(self, body, name, **kwargs):
+ def update(self, body, name, token=None, **kwargs):
raise NotImplementedError()
- def delete(self, name, **kwargs):
+ def delete(self, name, token=None, **kwargs):
raise NotImplementedError()
- def list(self, **kwargs):
+ def list(self, token=None, **kwargs):
raise NotImplementedError()
- def show(self, name, **kwargs):
+ def show(self, name, token=None, **kwargs):
raise NotImplementedError()
@@ -144,26 +170,25 @@ class ResourceApiBase(APIbase):
super().__init__(state)
self._type_name = type_name
- def list(self, **kwargs):
- resources = self._trans.get_resources(self._type_name)
+ def list(self, token=None, **kwargs):
+ resources = self._trans.get_resources(token, self._type_name)
return {
self._type_name: [r.as_dict() for r in resources]
}
- def show(self, name, **kwargs):
- return self._trans.get_resource(self._type_name, name).as_dict()
+ def show(self, name, token=None, **kwargs):
+ return self._trans.get_resource(token, self._type_name, name).as_dict()
- def create(self, body, **kwargs):
+ def create(self, body, token=None, **kwargs):
ick2.log.log(
'trace', msg_text='create resource',
- resource_type=self._type_name, body=body, kwargs=kwargs)
+ resource_type=self._type_name,
+ body=body, token=token, kwargs=kwargs)
as_dict = self.mangle_new_resource(body)
- rid = self.get_resource_name(as_dict)
- if self._trans.has_resource(self._type_name, rid):
- raise ick2.ExistsAlready(rid)
+ name = self.get_resource_name(as_dict)
- with self._trans.new(self._type_name, rid) as resource:
+ with self._trans.new(token, self._type_name, name) as resource:
resource.from_dict(as_dict)
return as_dict
@@ -174,12 +199,10 @@ class ResourceApiBase(APIbase):
def get_resource_name(self, resource): # pragma: no cover
raise NotImplementedError()
- def update(self, body, name, **kwargs):
- rid = self.get_resource_name(body)
- if not self._trans.has_resource(self._type_name, rid):
- raise ick2.NotFound(kind=self._type_name, rid=rid)
+ def update(self, body, name, token=None, **kwargs):
+ name = self.get_resource_name(body)
- with self._trans.modify(self._type_name, rid) as resource:
+ with self._trans.modify(token, self._type_name, name) as resource:
as_dict = self.mangle_updated_resource(resource.as_dict(), body)
resource.from_dict(as_dict)
@@ -188,7 +211,5 @@ class ResourceApiBase(APIbase):
def mangle_updated_resource(self, old, new): # pragma: no cover
return new
- def delete(self, name, **kwargs):
- if not self._trans.has_resource(self._type_name, name):
- raise ick2.NotFound(kind=self._type_name, rid=name)
- self._trans.remove_resource(self._type_name, name)
+ def delete(self, name, token=None, **kwargs):
+ self._trans.remove_resource(token, self._type_name, name)
diff --git a/ick2/buildsapi.py b/ick2/buildsapi.py
index 8862c63..2efc3d1 100644
--- a/ick2/buildsapi.py
+++ b/ick2/buildsapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -60,7 +60,7 @@ class BuildsAPI(ick2.ResourceApiBase): # pragma: no cover
raise ick2.MethodNotAllowed('Updating builds directly is not allowed')
def list(self, **kwargs):
- result = super().list()
+ result = super().list(**kwargs)
items = result[self._type_name]
items.sort(key=lambda x: x.get('build_number'))
result[self._type_name] = items
diff --git a/ick2/controllerapi.py b/ick2/controllerapi.py
index b28e1f7..a673e0d 100644
--- a/ick2/controllerapi.py
+++ b/ick2/controllerapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -18,8 +18,9 @@ import ick2
class ControllerAPI:
- def __init__(self, state):
+ def __init__(self, state, token_getter):
self._state = state
+ self._token_getter = token_getter
self._apis = {}
def set_apt_server(self, domain): # pragma: no cover
@@ -30,6 +31,7 @@ class ControllerAPI:
def set_auth_url(self, url): # pragma: no cover
self._set_url('set_auth_url', url)
+ self._token_getter.set_auth_url(url)
def set_notify_url(self, url): # pragma: no cover
self._set_url('set_notify_url', url)
@@ -60,7 +62,9 @@ class ControllerAPI:
for path in apis:
if path not in self._apis:
- self._apis[path] = apis[path](self._state)
+ api = apis[path](self._state)
+ api.set_token_getter(self._token_getter)
+ self._apis[path] = api
routes = []
for path, api in self._apis.items():
diff --git a/ick2/exceptions.py b/ick2/exceptions.py
index 3af7ff0..8467e7a 100644
--- a/ick2/exceptions.py
+++ b/ick2/exceptions.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -18,12 +18,34 @@ class IckException(Exception):
pass
+class NotFound(Exception):
+
+ def __init__(self, kind, name):
+ super().__init__(
+ 'Resource {}:{} not found'.format(
+ kind or "unknown", name or "unknown"))
+
+
class ExistsAlready(IckException):
def __init__(self, name):
super().__init__('Resource {} already exists'.format(name))
+class Conflict(IckException):
+
+ def __init__(self, rid, expected, got):
+ super().__init__(
+ 'Update conflict for {}: expected revision {}, got {}'.format(
+ rid, expected, got))
+
+
+class StoreError(IckException):
+
+ def __init__(self, msg):
+ super().__init__('Error accessing persistent store: {}'.format(msg))
+
+
class BadUpdate(IckException):
def __init__(self, how):
diff --git a/ick2/logapi.py b/ick2/logapi.py
index c43e8a6..16aeaef 100644
--- a/ick2/logapi.py
+++ b/ick2/logapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -59,7 +59,7 @@ class LogAPI(ick2.ResourceApiBase): # pragma: no cover
def update(self, body, name, **kwargs): # pragma: no cover
raise ick2.MethodNotAllowed('Updating builds directly is not allowed')
- def show(self, name, **kwargs):
- log = self._trans.get_resource('log', str(name))
+ def show(self, name, token=None, **kwargs):
+ log = self._trans.get_resource(token, 'log', str(name))
ick2.log.log('info', msg_text='Returning log', log=log.as_dict())
return log['log']
diff --git a/ick2/persistent.py b/ick2/persistent.py
deleted file mode 100644
index c5e2840..0000000
--- a/ick2/persistent.py
+++ /dev/null
@@ -1,156 +0,0 @@
-# Copyright (C) 2018 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import copy
-import os
-import urllib.parse
-
-
-import yaml
-
-
-import ick2
-
-
-class PersistentStateInterface: # pragma: no cover
-
- def get_resource_kinds(self):
- raise NotImplementedError()
-
- def get_resource_ids(self, kind):
- raise NotImplementedError()
-
- def has_resource(self, kind, rid):
- raise NotImplementedError()
-
- def get_resource(self, kind, rid):
- raise NotImplementedError()
-
- def get_resources(self, kind):
- return [
- self.get_resource(kind, rid)
- for rid in self.get_resource_ids(kind)
- ]
-
- def write_resource(self, kind, rid, resource):
- raise NotImplementedError()
-
- def remove_resource(self, kind, rid):
- raise NotImplementedError()
-
-
-class FilePersistentState(PersistentStateInterface):
-
- def __init__(self):
- self._dir = None
-
- def get_directory(self):
- return self._dir
-
- def set_directory(self, dirname):
- self._dir = dirname
-
- def _safe(self, name):
- return urllib.parse.quote(name, safe='')
-
- def _unsafe(self, safe):
- return urllib.parse.unquote(safe)
-
- def _unsafe_list(self, safe_names):
- return [self._unsafe(safe) for safe in safe_names]
-
- def _dirname(self, kind):
- return os.path.join(self._dir, self._safe(kind))
-
- def _filename(self, kind, rid):
- dirname = self._dirname(kind)
- return os.path.join(dirname, self._safe(rid))
-
- def get_resource_kinds(self):
- return self._unsafe_list(os.listdir(self._dir))
-
- def has_resource(self, kind, rid):
- filename = self._filename(kind, rid)
- return os.path.exists(filename)
-
- def get_resource_ids(self, kind):
- dirname = self._dirname(kind)
- if os.path.exists(dirname):
- return self._unsafe_list(os.listdir(dirname))
- return []
-
- def get_resource(self, kind, rid):
- filename = self._filename(kind, rid)
- if not os.path.exists(filename):
- raise ick2.NotFound(kind=kind, rid=rid)
- with open(filename, 'r') as f:
- as_dict = yaml.load(f, Loader=yaml.CSafeLoader)
- return resource_from_dict(as_dict)
-
- def write_resource(self, kind, rid, resource):
- dirname = self._dirname(kind)
- if not os.path.exists(dirname):
- os.makedirs(dirname)
-
- filename = self._filename(kind, rid)
- with open(filename, 'w') as f:
- yaml.dump(
- resource.as_dict(), stream=f, Dumper=yaml.CSafeDumper)
-
- def remove_resource(self, kind, rid):
- filename = self._filename(kind, rid)
- os.remove(filename)
-
-
-class NotFound(Exception):
-
- def __init__(self, kind, rid):
- super().__init__(
- 'Resource {}:{} not found'.format(
- kind or "unknown", rid or "unknown"))
-
-
-class Resource: # pragma: no cover
-
- def __init__(self, as_dict=None):
- self._dict = copy.deepcopy(as_dict or {})
-
- def as_dict(self):
- return copy.deepcopy(self._dict)
-
- def __getitem__(self, key):
- return self._dict[key]
-
- def __setitem__(self, key, value):
- self._dict[key] = value
-
- def __contains__(self, key):
- return key in self._dict
-
- def __len__(self):
- return len(self._dict)
-
- def get(self, key, default=None):
- return self._dict.get(key, default)
-
- def from_dict(self, as_dict):
- self._dict.clear()
- for key in as_dict:
- self[key] = as_dict[key]
-
-
-def resource_from_dict(as_dict): # pragma: no cover
- return Resource(as_dict)
diff --git a/ick2/persistent_tests.py b/ick2/persistent_tests.py
deleted file mode 100644
index 8acb141..0000000
--- a/ick2/persistent_tests.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Copyright (C) 2018 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import shutil
-import tempfile
-import unittest
-
-
-import ick2
-
-
-class FilePersistentStateTests(unittest.TestCase):
-
- def setUp(self):
- self.tempdir = tempfile.mkdtemp()
- self.state = ick2.FilePersistentState()
- self.state.set_directory(self.tempdir)
-
- def tearDown(self):
- shutil.rmtree(self.tempdir)
-
- def test_returns_dirname(self):
- self.assertEqual(self.state.get_directory(), self.tempdir)
-
- def test_has_no_resource_kinds_initially(self):
- self.assertEqual(self.state.get_resource_kinds(), [])
-
- def test_has_no_resources_initially(self):
- self.assertEqual(self.state.get_resource_ids('silly'), [])
-
- def test_has_no_resource_initially(self):
- with self.assertRaises(ick2.NotFound):
- self.state.get_resource('silly', '#1')
-
- def test_creates_resource(self):
- as_dict = {'foo': 'bar'}
- r = ick2.resource_from_dict(as_dict)
- self.state.write_resource('silly', '#1', r)
- self.assertTrue(self.state.has_resource('silly', '#1'))
- self.assertEqual(self.state.get_resource_kinds(), ['silly'])
- self.assertEqual(self.state.get_resource_ids('silly'), ['#1'])
-
- r2 = self.state.get_resource('silly', '#1')
- self.assertTrue(isinstance(r2, ick2.Resource))
- self.assertEqual(r.as_dict(), r2.as_dict())
-
- def test_removes_resource(self):
- as_dict = {'foo': 'bar'}
- r = ick2.resource_from_dict(as_dict)
- self.state.write_resource('silly', '#1', r)
- self.state.remove_resource('silly', '#1')
- self.assertFalse(self.state.has_resource('silly', '#1'))
- self.assertEqual(self.state.get_resource_ids('silly'), [])
diff --git a/ick2/projectapi.py b/ick2/projectapi.py
index 0d9f8f5..00fce8a 100644
--- a/ick2/projectapi.py
+++ b/ick2/projectapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -50,16 +50,17 @@ class ProjectAPI(ick2.ResourceApiBase):
]
def trigger_project(self, project, **kwargs): # pragma: no cover
- with self._trans.modify('projects', project) as p:
- self._start_build(p)
+ token = self._token_getter.get_token()
+ with self._trans.modify(token, 'projects', project) as p:
+ self._start_build(token, p)
return {'status': ick2.BUILD_TRIGGERED}
- def _start_build(self, project): # pragma: no cover
+ def _start_build(self, token, project): # pragma: no cover
build_no = self._pick_build_number(project)
build_id = '{}/{}'.format(project['project'], build_no)
ick2.log.log('info', msg_text='Starting new build', build_id=build_id)
- with self._trans.new('builds', build_id) as build:
+ with self._trans.new(token, 'builds', build_id) as build:
parameters = project.get('parameters', {})
build.from_dict({
'build_id': build_id,
@@ -81,10 +82,10 @@ class ProjectAPI(ick2.ResourceApiBase):
build_obj = ick2.Build(build)
graph = build_obj.get_graph()
graph.append_action(create_workspace)
- for action in self._get_actions(project):
+ for action in self._get_actions(token, project):
graph.append_action(action)
- with self._trans.new('log', build_id) as r:
+ with self._trans.new(token, 'log', build_id) as r:
r.from_dict({
'build_id': build_id,
'log': '',
@@ -101,11 +102,12 @@ class ProjectAPI(ick2.ResourceApiBase):
old_build_no=old_build_no, build_no=build_no)
return build_no
- def _get_actions(self, project): # pragma: no cover
+ def _get_actions(self, token, project): # pragma: no cover
actions = []
params = project.get('parameters', {})
for pipeline_name in project.get('pipelines', []):
- pipeline = self._trans.get_resource('pipelines', pipeline_name)
+ pipeline = self._trans.get_resource(
+ token, 'pipelines', pipeline_name)
wanted = pipeline.get('parameters', [])
missing = [
name
diff --git a/ick2/projectapi_tests.py b/ick2/projectapi_tests.py
index b6ec9e9..bdb58d1 100644
--- a/ick2/projectapi_tests.py
+++ b/ick2/projectapi_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -14,30 +14,31 @@
import os
-import shutil
-import tempfile
import unittest
import ick2
+class DummyTokenGetter:
+
+ def get_token(self):
+ return 'DUMMY.TOKEN'
+
+
class ProjectAPITests(unittest.TestCase):
def setUp(self):
- self.tempdir = tempfile.mkdtemp()
- self.statedir = os.path.join(self.tempdir, 'state/dir')
- self.state = ick2.FilePersistentState()
- self.state.set_directory(self.statedir)
-
- def tearDown(self):
- shutil.rmtree(self.tempdir)
+ self.state = ick2.MemoryStore()
def create_api(self):
return ick2.ProjectAPI(self.state)
def create_pipeline_api(self):
- return ick2.PipelineAPI(self.state)
+ getter = DummyTokenGetter()
+ api = ick2.PipelineAPI(self.state)
+ api.set_token_getter(getter)
+ return api
def test_has_not_projects_initially(self):
api = self.create_api()
diff --git a/ick2/resource.py b/ick2/resource.py
new file mode 100644
index 0000000..ca02d30
--- /dev/null
+++ b/ick2/resource.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2018-2019 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import copy
+import os
+import urllib.parse
+
+
+import yaml
+
+
+import ick2
+
+
+class Resource: # pragma: no cover
+
+ def __init__(self, as_dict=None):
+ self._dict = copy.deepcopy(as_dict or {})
+
+ def as_dict(self):
+ return copy.deepcopy(self._dict)
+
+ def __getitem__(self, key):
+ return self._dict[key]
+
+ def __setitem__(self, key, value):
+ self._dict[key] = value
+
+ def __contains__(self, key):
+ return key in self._dict
+
+ def __len__(self):
+ return len(self._dict)
+
+ def get(self, key, default=None):
+ return self._dict.get(key, default)
+
+ def from_dict(self, as_dict):
+ self._dict.clear()
+ for key in as_dict:
+ self[key] = as_dict[key]
+
+
+def resource_from_dict(as_dict): # pragma: no cover
+ return Resource(as_dict)
diff --git a/ick2/store.py b/ick2/store.py
new file mode 100644
index 0000000..83a008b
--- /dev/null
+++ b/ick2/store.py
@@ -0,0 +1,160 @@
+# Copyright (C) 2019 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import copy
+import json
+import uuid
+
+
+import requests
+
+
+import ick2
+
+
+class StoreInterface: # pragma: no cover
+
+ def create(self, token, obj):
+ raise NotImplementedError()
+
+ def update(self, token, rid, obj, revision):
+ raise NotImplementedError()
+
+ def show(self, token, rid):
+ raise NotImplementedError()
+
+ def delete(self, token, rid):
+ raise NotImplementedError()
+
+ def search(self, token, cond):
+ raise NotImplementedError()
+
+
+class MemoryStore(StoreInterface):
+
+ def __init__(self):
+ self._objs = {}
+
+ def search(self, token, cond):
+ return list(self._objs.keys())
+
+ def _new_id(self):
+ return str(uuid.uuid4())
+
+ def _set(self, rid, rev, obj):
+ self._objs[rid] = (rev, copy.deepcopy(obj))
+
+ def create(self, token, obj):
+ rid = self._new_id()
+ rev = self._new_id()
+ self._set(rid, rev, obj)
+ return rid, rev
+
+ def show(self, token, rid):
+ if rid not in self._objs:
+ raise ick2.NotFound('unknown.type', rid)
+ rev, obj = self._objs[rid]
+ return copy.deepcopy(obj), rev
+
+ def update(self, token, rid, obj, revision):
+ old_obj, old_rev = self.show(token, rid)
+ if old_rev != revision:
+ raise ick2.Conflict(rid, old_rev, revision)
+ new_rev = self._new_id()
+ self._set(rid, new_rev, obj)
+ return new_rev
+
+ def delete(self, token, rid):
+ del self._objs[rid]
+
+
+class MuckStore(StoreInterface): # pragma: no cover
+
+ def __init__(self, muck_url):
+ self._url = muck_url
+ ick2.log.log('info', msg_text='MuckStore created', muck_url=muck_url)
+
+ def _request(self, func, path, token, headers=None, body=None):
+ url = '{}{}'.format(self._url, path)
+ if headers is None:
+ headers = {}
+ headers['Authorization'] = 'Bearer {}'.format(token)
+ r = func(url, headers=headers, data=body)
+ ick2.log.log(
+ 'trace', msg_text='Accessing Muck',
+ func=repr(func), url=url, path=path, headers=headers, data=body,
+ status=r.status_code, text=r.text)
+ if not r.ok:
+ raise ick2.StoreError(r.text)
+ return r
+
+ def search(self, token, cond):
+ headers = {
+ 'Content-Type': 'application/json',
+ }
+ cond = {
+ 'cond': [
+ {
+ 'where': 'meta',
+ 'field': 'id',
+ 'op': '>=',
+ 'pattern': '',
+ }
+ ],
+ }
+ body = json.dumps(cond)
+ r = self._request(
+ requests.get, '/search', token, headers=headers, body=body)
+ obj = r.json()
+ return obj['resources']
+
+ def create(self, token, obj):
+ headers = {
+ 'Content-Type': 'application/json',
+ }
+ body = json.dumps(obj)
+ r = self._request(
+ requests.post, '/res', token, headers=headers, body=body)
+ rid = r.headers['Muck-Id']
+ rev = r.headers['Muck-Revision']
+ return rid, rev
+
+ def show(self, token, rid):
+ headers = {
+ 'Muck-Id': rid,
+ }
+ r = self._request(requests.get, '/res', token, headers=headers)
+ rev = r.headers['Muck-Revision']
+ as_dict = r.json()
+ return as_dict, rev
+
+ def update(self, token, rid, obj, revision):
+ headers = {
+ 'Content-Type': 'application/json',
+ 'Muck-Id': rid,
+ 'Muck-Revision': revision,
+ }
+ body = json.dumps(obj)
+ r = self._request(
+ requests.put, '/res', token, headers=headers, body=body)
+ rev = r.headers['Muck-Revision']
+ return rev
+
+ def delete(self, token, rid):
+ headers = {
+ 'Muck-Id': rid,
+ }
+ self._request(requests.delete, '/res', token, headers=headers)
diff --git a/ick2/store_tests.py b/ick2/store_tests.py
new file mode 100644
index 0000000..652f7f2
--- /dev/null
+++ b/ick2/store_tests.py
@@ -0,0 +1,89 @@
+# Copyright (C) 2019 Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import shutil
+import tempfile
+import unittest
+
+
+import ick2
+
+
+class StoreTests(unittest.TestCase):
+
+ def setUp(self):
+ self.store = ick2.MemoryStore()
+ self.token = 'dummy-test-token'
+ self.obj = {
+ 'foo': 'bar',
+ }
+
+ def create(self):
+ return self.store.create(self.token, self.obj)
+
+ def find_all_resources(self):
+ cond = [
+ {
+ 'where': 'meta',
+ 'field': 'id',
+ 'op': '!=',
+ 'pattern': '',
+ }
+ ]
+ return self.store.search(self.token, cond)
+
+ def test_has_no_resources_initially(self):
+ self.assertEqual(self.find_all_resources(), [])
+
+ def test_creates_resource(self):
+ rid, rev = self.create()
+ self.assertTrue(isinstance(rid, str))
+ self.assertTrue(isinstance(rev, str))
+ self.assertTrue(rid)
+ self.assertTrue(rev)
+ self.assertEqual(self.find_all_resources(), [rid])
+
+ def test_retrieves_resource(self):
+ rid, rev = self.create()
+ obj, rev = self.store.show(self.token, rid)
+ self.assertEqual(obj, self.obj)
+ self.assertFalse(obj is self.obj)
+
+ def test_retrieving_nonexistent_resource_raises_error(self):
+ with self.assertRaises(ick2.NotFound):
+ self.store.show(self.token, 'wrong.id')
+
+ def test_updating_with_wrong_revision_raises_error(self):
+ rid, rev = self.create()
+ with self.assertRaises(ick2.Conflict):
+ self.store.update(self.token, rid, self.obj, 'wrong.revision')
+
+ def test_updates_resource(self):
+ rid, rev = self.create()
+ obj2 = {
+ 'yo': 'yoyo',
+ }
+ rev2 = self.store.update(self.token, rid, obj2, rev)
+ obj3, rev3 = self.store.show(self.token, rid)
+ self.assertTrue(rev2)
+ self.assertNotEqual(rev, rev2)
+ self.assertEqual(rev2, rev3)
+ self.assertEqual(obj2, obj3)
+
+ def test_deletes_resource(self):
+ rid, rev = self.create()
+ self.store.delete(self.token, rid)
+ self.assertEqual(self.find_all_resources(), [])
diff --git a/ick2/tokengetter.py b/ick2/tokengetter.py
new file mode 100644
index 0000000..c152148
--- /dev/null
+++ b/ick2/tokengetter.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2019 Lars Wirzenius
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import time
+
+
+import jwt
+
+
+import ick2
+
+
+class TokenGetter:
+
+ scopes = [
+ 'super',
+ 'create',
+ 'update',
+ 'show',
+ 'delete',
+ 'uapi_workers_post',
+ 'uapi_workers_id_get',
+ 'uapi_workers_id_put',
+ 'uapi_workers_id_delete',
+ 'uapi_builds_post',
+ 'uapi_builds_id_get',
+ 'uapi_builds_id_put',
+ 'uapi_builds_id_delete',
+ 'uapi_logs_post',
+ 'uapi_logs_id_get',
+ 'uapi_logs_id_put',
+ 'uapi_logs_id_delete',
+ ]
+
+ def __init__(self, client_id, client_secret):
+ self._ac = ick2.AuthClient()
+ self._ac.set_client_creds(client_id, client_secret)
+ self._token = None
+ self._token_exp = None
+
+ def set_auth_url(self, auth_url):
+ self._ac.set_auth_url(auth_url)
+
+ def get_token(self):
+ if not self._got_valid_token():
+ self._get_new_token()
+ return self._token
+
+ def _got_valid_token(self):
+ fuzz = 10
+ return (self._token is not None and
+ self._token_exp is not None and
+ time.time() + fuzz < self._token_exp)
+
+ def _get_new_token(self):
+ self._token = self._ac.get_token(' '.join(self.scopes))
+ parsed = jwt.decode(self._token, verify=False)
+ self._token_exp = parsed['exp']
diff --git a/ick2/trans.py b/ick2/trans.py
index f1c8e5e..da2f80d 100644
--- a/ick2/trans.py
+++ b/ick2/trans.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Lars Wirzenius
+# Copyright (C) 2018-2019 Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -14,19 +14,75 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import copy
+
+
import ick2
+def wrap(kind, name, as_dict):
+ return {
+ 'kind': kind,
+ 'name': name,
+ 'res': copy.deepcopy(as_dict),
+ }
+
+
+def unwrap(obj):
+ return obj['kind'], obj['name'], copy.deepcopy(obj['res'])
+
+
+def find_by_name(store, token, kind, name):
+ rids = store.search(token, None) # FIXME for real searches
+ for rid in rids:
+ obj, rev = store.show(token, rid)
+ if obj is not None:
+ okind, oname, as_dict = unwrap(obj)
+ if okind == kind and oname == name:
+ return as_dict, rid, rev
+
+ raise ick2.NotFound(kind=kind, name=name)
+
+
+def find_names_by_kind(store, token, kind):
+ rids = store.search(token, None) # FIXME for real searches
+ for rid in rids:
+ obj, rev = store.show(token, rid)
+ if obj is not None:
+ okind, oname, as_dict = unwrap(obj)
+ if okind == kind:
+ yield oname
+
+
class TransactionalResource:
- def __init__(self, state, kind, rid):
- self.state = state
+ def __init__(self, new, token, store, kind, name):
+ self.token = token
+ self.store = store
self.kind = kind
- self.rid = rid
- if state.has_resource(kind, rid):
- self.resource = state.get_resource(kind, rid)
+ self.name = name
+
+ self.rid = None
+ self.rev = None
+ self.resource = None
+
+ if new:
+ try:
+ find_by_name(self.store, token, kind, name)
+ except ick2.NotFound:
+ as_dict = {}
+ else:
+ raise ick2.ExistsAlready(name)
else:
- self.resource = ick2.resource_from_dict({})
+ try:
+ as_dict, self.rid, self.rev = find_by_name(
+ self.store, token, kind, name)
+ except ick2.NotFound:
+ raise
+
+ self.resource = ick2.resource_from_dict(as_dict)
+ assert ((new and self.rid is None) or
+ (not new and self.rid is not None))
methods = [
'as_dict',
@@ -43,7 +99,12 @@ class TransactionalResource:
def __exit__(self, exc_type, value, traceback):
if exc_type is None:
- self.state.write_resource(self.kind, self.rid, self.resource)
+ as_dict = self.resource.as_dict()
+ obj = wrap(self.kind, self.name, as_dict)
+ if self.rid is None:
+ self.store.create(self.token, obj)
+ else:
+ self.store.update(self.token, self.rid, obj, self.rev)
class TransactionalState:
@@ -51,28 +112,22 @@ class TransactionalState:
def __init__(self, state):
self.state = state
- def new(self, kind, rid):
- return TransactionalResource(self.state, kind, rid)
+ def new(self, token, kind, name):
+ return TransactionalResource(True, token, self.state, kind, name)
- def modify(self, kind, rid):
- if not self.state.has_resource(kind, rid):
- raise ick2.NotFound(kind=kind, rid=rid)
- return TransactionalResource(self.state, kind, rid)
+ def modify(self, token, kind, name):
+ return TransactionalResource(False, token, self.state, kind, name)
- def get_resource_kinds(self):
- return self.state.get_resource_kinds()
+ def get_resource(self, token, kind, name):
+ as_dict, rid, rev = find_by_name(self.state, token, kind, name)
+ return ick2.resource_from_dict(as_dict)
- def get_resource_ids(self, kind):
- return self.state.get_resource_ids(kind)
-
- def has_resource(self, kind, rid):
- return self.state.has_resource(kind, rid)
-
- def get_resource(self, kind, rid):
- return self.state.get_resource(kind, rid)
-
- def get_resources(self, kind):
- return self.state.get_resources(kind)
+ def get_resources(self, token, kind):
+ return [
+ self.get_resource(token, kind, name)
+ for name in find_names_by_kind(self.state, token, kind)
+ ]
- def remove_resource(self, kind, rid):
- self.state.remove_resource(kind, rid)
+ def remove_resource(self, token, kind, name):
+ as_dict, rid, rev = find_by_name(self.state, token, kind, name)
+ self.state.delete(token, rid)
diff --git a/ick2/version.py b/ick2/version.py
index 4c6e147..e69102d 100644
--- a/ick2/version.py
+++ b/ick2/version.py
@@ -1,2 +1,2 @@
-__version__ = "0.53.2+git"
-__version_info__ = (0, 53, 2, '+git')
+__version__ = "0.54+git"
+__version_info__ = (0, 54, '+git')
diff --git a/ick2/workapi.py b/ick2/workapi.py
index 5179f90..7ff6f93 100644
--- a/ick2/workapi.py
+++ b/ick2/workapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -40,24 +40,25 @@ class WorkAPI(ick2.APIbase):
},
]
- def get_work(self, **kwargs):
+ def get_work(self, token=None, **kwargs):
+ token = self._token_getter.get_token()
worker_id = self._get_client_id(**kwargs)
ick2.log.log(
'trace', msg_text='Worker wants work', worker_id=worker_id)
- with self._trans.modify('workers', worker_id) as worker:
+ with self._trans.modify(token, 'workers', worker_id) as worker:
doing = worker.get('doing')
if doing:
ick2.log.log(
'trace', msg_text='Worker already got work', doing=doing)
return doing
- build_id = self._pick_build(worker_id)
+ build_id = self._pick_build(token, worker_id)
if build_id is None:
ick2.log.log('trace', msg_text='No suitable build for worker')
return {}
- with self._trans.modify('builds', build_id) as build:
- with self._trans.modify('log', build_id) as log:
+ with self._trans.modify(token, 'builds', build_id) as build:
+ with self._trans.modify(token, 'log', build_id) as log:
ick2.log.log(
'trace', msg_text='Picked build for worker',
build_id=build_id, build=build.as_dict())
@@ -115,7 +116,7 @@ class WorkAPI(ick2.APIbase):
raise ick2.ClientIdMissing()
return client_id
- def _pick_build(self, worker):
+ def _pick_build(self, token, worker):
def on_worker(build):
return build.get('worker') == worker
@@ -129,7 +130,7 @@ class WorkAPI(ick2.APIbase):
def is_triggered(build):
return status(build) == ick2.BUILD_TRIGGERED
- builds = self._trans.get_resources('builds')
+ builds = self._trans.get_resources(token, 'builds')
return (self._find_build(builds, on_worker, is_building) or
self._find_build(builds, is_triggered))
@@ -139,7 +140,9 @@ class WorkAPI(ick2.APIbase):
return build['build_id']
return None
- def update_work(self, update, **kwargs):
+ def update_work(self, update, token=None, **kwargs):
+ token = self._token_getter.get_token()
+
try:
worker_id = update['worker']
build_id = update['build_id']
@@ -148,9 +151,9 @@ class WorkAPI(ick2.APIbase):
except KeyError as e: # pragma: no cover
raise ick2.BadUpdate(str(e))
- with self._trans.modify('workers', worker_id) as worker:
- with self._trans.modify('builds', build_id) as build:
- with self._trans.modify('log', build_id) as log:
+ with self._trans.modify(token, 'workers', worker_id) as worker:
+ with self._trans.modify(token, 'builds', build_id) as build:
+ with self._trans.modify(token, 'log', build_id) as log:
doing = worker.get('doing', {})
self._check_work_update(doing, update)
self._append_to_build_log(log, update)
diff --git a/ick2/workapi_tests.py b/ick2/workapi_tests.py
index c368b4b..0ac3a86 100644
--- a/ick2/workapi_tests.py
+++ b/ick2/workapi_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -15,8 +15,6 @@
import copy
import os
-import shutil
-import tempfile
import unittest
@@ -26,15 +24,9 @@ import ick2
class WorkAPITests(unittest.TestCase):
def setUp(self):
- self.tempdir = tempfile.mkdtemp()
- self.statedir = os.path.join(self.tempdir, 'state/dir')
- self.state = ick2.FilePersistentState()
- self.state.set_directory(self.statedir)
+ self.state = ick2.MemoryStore()
self.claims = None
- def tearDown(self):
- shutil.rmtree(self.tempdir)
-
def create_project_api(self):
pipeline = {
'pipeline': 'build',
@@ -44,7 +36,9 @@ class WorkAPITests(unittest.TestCase):
],
}
+ getter = DummyTokenGetter()
pipeapi = ick2.PipelineAPI(self.state)
+ pipeapi.set_token_getter(getter)
pipeapi.create(pipeline)
project = {
@@ -55,6 +49,7 @@ class WorkAPITests(unittest.TestCase):
'pipelines': ['build'],
}
api = ick2.ProjectAPI(self.state)
+ api.set_token_getter(getter)
api.create(project)
return api
@@ -65,12 +60,17 @@ class WorkAPITests(unittest.TestCase):
self.claims = {
'aud': 'asterix',
}
+ getter = DummyTokenGetter()
api = ick2.WorkerAPI(self.state)
+ api.set_token_getter(getter)
api.create(worker, claims=self.claims)
return api
def create_work_api(self):
- return ick2.WorkAPI(self.state)
+ getter = DummyTokenGetter()
+ api = ick2.WorkAPI(self.state)
+ api.set_token_getter(getter)
+ return api
def test_worker_gets_no_work_when_no_builds_have_been_triggered(self):
self.create_project_api()
@@ -295,3 +295,9 @@ class WorkAPITests(unittest.TestCase):
# Ask for work again.
self.assertEqual(work.get_work(claims=self.claims), {})
+
+
+class DummyTokenGetter:
+
+ def get_token(self):
+ return 'DUMMY.TOKEN'
diff --git a/ick2/workerapi.py b/ick2/workerapi.py
index d4b508c..9a2c5f2 100644
--- a/ick2/workerapi.py
+++ b/ick2/workerapi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -34,4 +34,5 @@ class WorkerAPI(ick2.ResourceApiBase): # pragma: no cover
def create(self, body, **kwargs):
client_id = self._get_client_id(**kwargs)
body['worker'] = client_id
+ kwargs['token'] = self._token_getter.get_token()
return super().create(body, **kwargs)
diff --git a/ick_controller.py b/ick_controller.py
index 2090b4e..7299e60 100644
--- a/ick_controller.py
+++ b/ick_controller.py
@@ -1,5 +1,5 @@
#!/usr/bin/python3
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -47,9 +47,12 @@ default_config = {
'log': [],
'statedir': None,
'artifact-store': None,
+ 'muck-url': None,
'auth-url': None,
'notify-url': None,
'apt-server': None,
+ 'client-id': None,
+ 'client-secret': None,
}
@@ -93,10 +96,11 @@ def main():
ick2.log.log('info', msg_text='Ick2 controller starts', config=config)
- state = ick2.FilePersistentState()
- state.set_directory(config['statedir'])
+ state = ick2.MuckStore(config['muck-url'])
- api = ick2.ControllerAPI(state)
+ getter = ick2.TokenGetter(config['client-id'], config['client-secret'])
+
+ api = ick2.ControllerAPI(state, getter)
api.set_apt_server(config['apt-server'])
api.set_artifact_store_url(config['artifact-store'])
api.set_auth_url(config['auth-url'])
diff --git a/icktool b/icktool
index 64b0793..62ec215 100755
--- a/icktool
+++ b/icktool
@@ -1,5 +1,5 @@
#!/usr/bin/python3
-# Copyright 2017-2018 Lars Wirzenius
+# Copyright 2017-2019 Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -67,6 +67,10 @@ class Icktool(cliapp.Application):
'uapi_blobs_id_get',
'uapi_blobs_id_put',
'uapi_notify_post',
+ 'show',
+ 'create',
+ 'update',
+ 'delete',
] + scopes_for_types(types)
def add_settings(self):
diff --git a/notification_service.py b/notification_service.py
index bd6bad1..f8e5c55 100644
--- a/notification_service.py
+++ b/notification_service.py
@@ -1,5 +1,5 @@
#!/usr/bin/python3
-# Copyright (C) 2018 Lars Wirzenius
+# Copyright (C) 2018-2019 Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -123,4 +123,4 @@ app = main()
if __name__ == '__main__':
print('running in debug mode')
- app.run(host='127.0.0.1', port=12767)
+ app.run(host='127.0.0.1', port=6666)
diff --git a/start_ick b/start_ick
index 8c9d50f..9939e01 100755
--- a/start_ick
+++ b/start_ick
@@ -1,5 +1,5 @@
#!/bin/sh
-# Copyright (C) 2017-2018 Lars Wirzenius
+# Copyright (C) 2017-2019 Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@@ -26,7 +26,7 @@ else
fi
gunicorn3 \
- --bind 127.0.0.1:12765 \
+ --bind 127.0.0.1:3333 \
--log-file "$GUNICORN_LOG" \
--log-level debug \
ick_controller:app
diff --git a/without-tests b/without-tests
index 0e24420..673068e 100644
--- a/without-tests
+++ b/without-tests
@@ -9,7 +9,9 @@ ick2/logapi.py
ick2/logging.py
ick2/notificationapi.py
ick2/pipelineapi.py
+ick2/resource.py
ick2/responses.py
+ick2/tokengetter.py
ick2/trans.py
ick2/sendmail.py
ick2/version.py
diff --git a/yarns/100-projects.yarn b/yarns/100-projects.yarn
index 4c5291f..3c4edbc 100644
--- a/yarns/100-projects.yarn
+++ b/yarns/100-projects.yarn
@@ -53,6 +53,7 @@ building them. We start by starting an instance of the controller.
... uapi_projects_id_get
... uapi_projects_id_put
... uapi_projects_id_delete
+ ... create update show delete
AND a running ick controller
WHEN user makes request GET /projects
diff --git a/yarns/150-pipelines.yarn b/yarns/150-pipelines.yarn
index 6a303eb..7b3a339 100644
--- a/yarns/150-pipelines.yarn
+++ b/yarns/150-pipelines.yarn
@@ -64,6 +64,7 @@ running them. We start by starting an instance of the controller.
... uapi_pipelines_id_get
... uapi_pipelines_id_put
... uapi_pipelines_id_delete
+ ... create update show delete
AND a running ick controller
WHEN user makes request GET /pipelines
diff --git a/yarns/300-workers.yarn b/yarns/300-workers.yarn
index cea6c81..2375a31 100644
--- a/yarns/300-workers.yarn
+++ b/yarns/300-workers.yarn
@@ -57,6 +57,7 @@ controller API. It doesn't actually talk to the worker itself.
... uapi_workers_id_get
... uapi_workers_id_put
... uapi_workers_id_delete
+ ... create update show delete
AND an access token for obelix with scopes
... uapi_workers_post
AND a running ick controller
diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn
index 13eefce..08eb8f9 100644
--- a/yarns/400-build.yarn
+++ b/yarns/400-build.yarn
@@ -43,6 +43,7 @@ Set up the controller.
... uapi_logs_get
... uapi_logs_id_delete
... uapi_logs_id_get
+ ... create update show delete
AND a running ick controller
Add up a project with some named pipelines.
@@ -886,6 +887,7 @@ Set up the controller.
... uapi_logs_get
... uapi_logs_id_delete
... uapi_logs_id_get
+ ... create update show delete
AND a running ick controller
Add a couple of projects.
@@ -1119,6 +1121,7 @@ Set up the controller.
... uapi_logs_get
... uapi_logs_id_delete
... uapi_logs_id_get
+ ... create update show delete
AND a running ick controller
Add a couple of projects.
diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn
index 6ca06c6..08592a7 100644
--- a/yarns/500-build-fail.yarn
+++ b/yarns/500-build-fail.yarn
@@ -45,6 +45,7 @@ Set up the controller.
... uapi_logs_get
... uapi_logs_id_delete
... uapi_logs_id_get
+ ... create update show delete
AND a running ick controller
Add up a project and its pipelines.
diff --git a/yarns/600-unauthz.yarn b/yarns/600-unauthz.yarn
index ab33404..cca142e 100644
--- a/yarns/600-unauthz.yarn
+++ b/yarns/600-unauthz.yarn
@@ -37,6 +37,7 @@ Set up the controller.
... uapi_builds_get
... uapi_builds_id_get
... uapi_logs_id_get
+ ... create update show delete
AND a running ick controller
WHEN user makes request POST /projects with a valid token and body
diff --git a/yarns/700-artifact-store.yarn b/yarns/700-artifact-store.yarn
index cadc83c..e562957 100644
--- a/yarns/700-artifact-store.yarn
+++ b/yarns/700-artifact-store.yarn
@@ -31,6 +31,7 @@ Set up the artifact store.
... uapi_blobs_id_delete
... uapi_blobs_id_put
... uapi_blobs_id_get
+ ... create update show delete
AND a running ick controller
<!--