diff options
-rwxr-xr-x | check | 72 | ||||
-rwxr-xr-x | create-token | 47 | ||||
-rwxr-xr-x | generate-rsa-key | 34 | ||||
-rw-r--r-- | setup.py | 3 | ||||
-rw-r--r-- | yarns/000.yarn | 33 | ||||
-rw-r--r-- | yarns/100-projects.yarn | 9 | ||||
-rw-r--r-- | yarns/150-pipelines.yarn | 9 | ||||
-rw-r--r-- | yarns/200-version.yarn | 12 | ||||
-rw-r--r-- | yarns/300-workers.yarn | 33 | ||||
-rw-r--r-- | yarns/400-build.yarn | 209 | ||||
-rw-r--r-- | yarns/500-build-fail.yarn | 53 | ||||
-rw-r--r-- | yarns/600-unauthz.yarn | 15 | ||||
-rw-r--r-- | yarns/700-artifact-store.yarn | 29 | ||||
-rw-r--r-- | yarns/900-implements.yarn | 37 | ||||
-rw-r--r-- | yarns/900-local.yarn | 121 | ||||
-rw-r--r-- | yarns/900-remote.yarn | 21 | ||||
-rw-r--r-- | yarns/lib.py | 214 |
17 files changed, 409 insertions, 542 deletions
@@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2017-2018 Lars Wirzenius +# Copyright 2017-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -30,32 +30,16 @@ title() } -title Remote or local yarns? -remote=no -unit=yes -yarns=yes +title Remote yarns? +yarns=no if [ "$#" -gt 0 ] then case "$1" in https://*) - remote=yes - unit=no yarns=yes remote_url="$1" shift 1 ;; - yarns) - remote=no - unit=no - yarns=yes - shift 1 - ;; - local) - remote=no - unit=yes - yarns=no - shift 1 - ;; *) echo "Don't understand args: $@" 1>&2 exit 1 @@ -64,51 +48,41 @@ then fi -if [ "$unit" = yes ] -then - title Unit tests - python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 +title Unit tests +python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 - if [ -e .git ] - then - sources="$(git ls-files | grep -Fvxf copyright-exceptions)" +if [ -e .git ] +then + sources="$(git ls-files | grep -Fvxf copyright-exceptions)" - title Copyright statements - copyright-statement-lint $sources + title Copyright statements + copyright-statement-lint $sources - title Copyright licences - ./is-agpl3+ $sources - fi + title Copyright licences + ./is-agpl3+ $sources +fi - python_sources="ick_controller.py worker_manager ick2 icktool" +python_sources="ick_controller.py worker_manager ick2 icktool" - title pycodestyle - pycodestyle ick2 $python_sources +title pycodestyle +pycodestyle ick2 $python_sources - if command -v pylint3 > /dev/null - then - title pylint3 - pylint3 --rcfile pylint.conf $python_sources - fi +if command -v pylint3 > /dev/null +then + title pylint3 + pylint3 --rcfile pylint.conf $python_sources fi if [ "$yarns" = yes ] then title Yarns - if [ "$remote" = no ] - then - impl=yarns/900-local.yarn - args="" - else - impl=yarns/900-remote.yarn - args="--env ICK_URL=$remote_url" - fi - yarn yarns/[^9]*.yarn yarns/900-implements.yarn "$impl" \ + yarn yarns/*.yarn \ --shell python2 \ --shell-arg '' \ --shell-library yarns/lib.py \ --cd-datadir \ - $args \ + --env "CONTROLLER=$remote_url" \ + --env "SECRETS=$HOME/.config/qvarn/createtoken.conf" \ "$@" fi diff --git a/create-token b/create-token deleted file mode 100755 index 55a7f7e..0000000 --- a/create-token +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/python3 -# Copyright (C) 2017-2018 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import sys -import time - -import Crypto.PublicKey.RSA - -import apifw - - -# FIXME: These should agree with how ick controller is configured. -# See the Ansible playbook. -iss = 'localhost' - - -key_text = sys.stdin.read() -key = Crypto.PublicKey.RSA.importKey(key_text) - -scopes = ' '.join(sys.argv[1].split()) -aud = sys.argv[2] - -now = time.time() -claims = { - 'iss': iss, - 'sub': 'subject-uuid', - 'aud': aud, - 'exp': now + 86400, # FIXME: This is silly long - 'scope': scopes, -} - -token = apifw.create_token(claims, key) -sys.stdout.write(token.decode('ascii')) diff --git a/generate-rsa-key b/generate-rsa-key deleted file mode 100755 index e44a796..0000000 --- a/generate-rsa-key +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/python3 -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import sys - -import Crypto.PublicKey.RSA - - -RSA_KEY_BITS = 4096 # A nice, currently safe length - -key = Crypto.PublicKey.RSA.generate(RSA_KEY_BITS) - -filename = sys.argv[1] - -def write(filename, byts): - with open(filename, 'w') as f: - f.write(byts.decode('ascii')) - -write(filename, key.exportKey('PEM')) -write(filename + '.pub', key.exportKey('OpenSSH')) @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright (C) 2017-2018 Lars Wirzenius <liw@liw.fi> +# Copyright (C) 2017-2019 Lars Wirzenius <liw@liw.fi> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -53,7 +53,6 @@ setup( ], packages=['ick2'], scripts=[ - 'create-token', 'start_ick', 'start_artifact_store', 'start_notification_service', diff --git a/yarns/000.yarn b/yarns/000.yarn index d5db1a1..12bdea2 100644 --- a/yarns/000.yarn +++ b/yarns/000.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017 Lars Wirzenius +Copyright 2017,2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -34,30 +34,25 @@ system. Written for execution by [yarn][]. ## Running this test suite -This test suite tests an Ick2 controller, and can either run in a -local or remote mode. In local mode, each test scenario starts and -stops a local instance, and runs tests against that. In remote mode, -an existing, running controller instance is assumed, and tests are run -against that. +This test suite tests a deployed Ick2 controller and other components of Ick, +but not the workers. The deployed Ick2 must not have any workers, and +must be "empty", meaning, no project, pipelines, etc, must be defined. +The test suit deletes everything. -The `./check` script runs the tests. By default it runs in local mode. -Local mode can be specified explicitly with the `local` parameter: +The `./check` script runs the tests. It can run only local tests, +which are mainly unit tests and code health. EXAMPLE running the test suite in local mode ./check - ./check local - ./check local -v --tempdir tmp --snapshot `./check` can be given extra arguments, which it will pass on to -`yarn`. - -To run the tests in remote mode, give the controller URL: +`yarn` to test a remote Ick instance, which may not have workers. The +first argument is the controller URL: EXAMPLE running the test suite in local mode - ./check https://ick-controller --env ICK_PRIVATE_KEY=~/tmp/ick.key - ./check https://ick-controller --env ICK_PRIVATE_KEY=~/tmp/ick.key \ - -v --tempdir tmp --snapshot + ./check https://ick-controller + ./check https://ick-controller -v --tempdir tmp --snapshot -The URL **must** be an `https` URL. Additionally, the environment -variable `ICK_PRIVATE_KEY` must be given a path to the *private* key -for signing tokens, so that a new token can be generated. +The URL **must** be an `https` URL. `qvisqvetool` must be configured +to suppot the given Ick instance, so that test clients for API use can +be managed by yarn automatically. diff --git a/yarns/100-projects.yarn b/yarns/100-projects.yarn index b75dea4..4c5291f 100644 --- a/yarns/100-projects.yarn +++ b/yarns/100-projects.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -46,18 +46,13 @@ First we test the controller API for managing projects, without building them. We start by starting an instance of the controller. SCENARIO managing projects - GIVEN an RSA key pair for token signing - AND an access token for user with scopes + GIVEN an access token for user with scopes ... uapi_pipelines_post ... uapi_projects_get ... uapi_projects_post ... uapi_projects_id_get ... uapi_projects_id_put ... uapi_projects_id_delete - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify AND a running ick controller WHEN user makes request GET /projects diff --git a/yarns/150-pipelines.yarn b/yarns/150-pipelines.yarn index d828935..6a303eb 100644 --- a/yarns/150-pipelines.yarn +++ b/yarns/150-pipelines.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -58,17 +58,12 @@ First we test the controller API for managing pipelines, without running them. We start by starting an instance of the controller. SCENARIO managing pipelines - GIVEN an RSA key pair for token signing - AND an access token for user with scopes + GIVEN an access token for user with scopes ... uapi_pipelines_get ... uapi_pipelines_post ... uapi_pipelines_id_get ... uapi_pipelines_id_put ... uapi_pipelines_id_delete - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify AND a running ick controller WHEN user makes request GET /pipelines diff --git a/yarns/200-version.yarn b/yarns/200-version.yarn index 710a57a..7fceef2 100644 --- a/yarns/200-version.yarn +++ b/yarns/200-version.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -22,21 +22,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. The Ick controller reports is version upon request. SCENARIO checking controller version - GIVEN an RSA key pair for token signing - AND an access token for user with scopes + GIVEN an access token for user with scopes ... uapi_version_get - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify AND a running ick controller WHEN user makes request GET /version THEN result has status code 200 AND version in body matches version from setup.py - AND artifact store URL is https://blobs.example.com - AND authentication URL is https://auth.example.com - AND notify URL is https://notify.example.com FINALLY stop ick controller diff --git a/yarns/300-workers.yarn b/yarns/300-workers.yarn index 6399b20..cea6c81 100644 --- a/yarns/300-workers.yarn +++ b/yarns/300-workers.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -52,18 +52,13 @@ Note that this only tests managing information about workers via the controller API. It doesn't actually talk to the worker itself. SCENARIO managing workers - GIVEN an RSA key pair for token signing - AND an access token for user with scopes + GIVEN an access token for user with scopes ... uapi_workers_get ... uapi_workers_id_get ... uapi_workers_id_put ... uapi_workers_id_delete AND an access token for obelix with scopes ... uapi_workers_post - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify AND a running ick controller WHEN user makes request GET /workers @@ -72,7 +67,6 @@ controller API. It doesn't actually talk to the worker itself. WHEN obelix makes request POST /workers with a valid token and body ... { - ... "worker": "obelix", ... "protocol": "ssh", ... "address": "obelix.ick.example", ... "user": "ick", @@ -81,9 +75,10 @@ controller API. It doesn't actually talk to the worker itself. ... } ... } THEN result has status code 201 + AND worker id is OBELIX AND body matches ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "protocol": "ssh", ... "address": "obelix.ick.example", ... "user": "ick", @@ -99,7 +94,7 @@ controller API. It doesn't actually talk to the worker itself. ... { ... "workers": [ ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "protocol": "ssh", ... "address": "obelix.ick.example", ... "user": "ick", @@ -112,11 +107,11 @@ controller API. It doesn't actually talk to the worker itself. WHEN user stops ick controller GIVEN a running ick controller - WHEN user makes request GET /workers/obelix + WHEN user makes request GET /workers/${OBELIX} THEN result has status code 200 AND body matches ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "protocol": "ssh", ... "address": "obelix.ick.example", ... "user": "ick", @@ -125,10 +120,10 @@ controller API. It doesn't actually talk to the worker itself. ... } ... } - WHEN user makes request PUT /workers/obelix with a valid token + WHEN user makes request PUT /workers/${OBELIX} with a valid token ... and body ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "protocol": "local", ... "keywords": { ... "debian_codename": "unstable" @@ -137,7 +132,7 @@ controller API. It doesn't actually talk to the worker itself. THEN result has status code 200 AND body matches ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "protocol": "local", ... "keywords": { ... "debian_codename": "unstable" @@ -145,20 +140,20 @@ controller API. It doesn't actually talk to the worker itself. ... } AND controller state directory contains worker obelix - WHEN user makes request GET /workers/obelix + WHEN user makes request GET /workers/${OBELIX} THEN result has status code 200 AND body matches ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "protocol": "local", ... "keywords": { ... "debian_codename": "unstable" ... } ... } - WHEN user makes request DELETE /workers/obelix + WHEN user makes request DELETE /workers/${OBELIX} THEN result has status code 200 - WHEN user makes request GET /workers/obelix + WHEN user makes request GET /workers/${OBELIX} THEN result has status code 404 FINALLY stop ick controller diff --git a/yarns/400-build.yarn b/yarns/400-build.yarn index 5172ba0..13eefce 100644 --- a/yarns/400-build.yarn +++ b/yarns/400-build.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -25,20 +25,23 @@ This scenario tests the controller API to simulate a build. Set up the controller. - GIVEN an RSA key pair for token signing - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify - AND an access token for user with scopes + GIVEN an access token for user with scopes ... uapi_pipelines_post + ... uapi_pipelines_get + ... uapi_pipelines_id_delete ... uapi_projects_post + ... uapi_projects_get + ... uapi_projects_id_delete ... uapi_projects_id_status_put ... uapi_projects_id_status_get ... uapi_projects_id_builds_get + ... uapi_workers_id_delete ... uapi_workers_id_get ... uapi_builds_get + ... uapi_builds_id_delete ... uapi_builds_id_get + ... uapi_logs_get + ... uapi_logs_id_delete ... uapi_logs_id_get AND a running ick controller @@ -105,6 +108,7 @@ Register a worker. ... { ... } THEN result has status code 201 + AND worker id is OBELIX Trigger build of project that doesn't exist. @@ -135,7 +139,7 @@ the worker to construct a new workspace for the build. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -154,7 +158,7 @@ the worker to construct a new workspace for the build. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -168,16 +172,16 @@ the worker to construct a new workspace for the build. User can now see pipeline is running and which worker is building it. - WHEN user makes request GET /workers/obelix + WHEN user makes request GET /workers/${OBELIX} THEN result has status code 200 AND body matches ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "doing": { ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -199,7 +203,7 @@ User can now see pipeline is running and which worker is building it. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "graph": { ... "1": { @@ -238,7 +242,7 @@ Worker reports workspace creation is done. Note the zero exit code. ... { ... "build_id": "rome/1", ... "action_id": "1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "", @@ -256,7 +260,7 @@ Worker requests more work, and gets the first actual build step. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -275,7 +279,7 @@ hasn't finished yet. ... { ... "build_id": "rome/1", ... "action_id": "2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": null, ... "stdout": "hey ho", @@ -294,7 +298,7 @@ didn't finish. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -319,7 +323,7 @@ Report the step is done, and successfully. ... { ... "build_id": "rome/1", ... "action_id": "2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": ", hey ho\n", @@ -344,7 +348,7 @@ The build status now shows the next step as the active one. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -382,7 +386,7 @@ Now there's another step to do. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -396,15 +400,15 @@ Now there's another step to do. User sees changed status. - WHEN user makes request GET /workers/obelix + WHEN user makes request GET /workers/${OBELIX} THEN result has status code 200 AND body matches ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "doing": { ... "build_id": "rome/1", ... "build_number": 1, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -424,7 +428,7 @@ Report it done. ... { ... "build_id": "rome/1", ... "action_id": "3", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "to the gold mine we go!\n", @@ -441,7 +445,7 @@ Worker now gets told to notify about the build. ... { ... "build_id": "rome/1", ... "build_number": 1, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -459,7 +463,7 @@ Report it's done. ... { ... "build_id": "rome/1", ... "action_id": "4", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "", @@ -486,7 +490,7 @@ current action. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -526,7 +530,7 @@ current action. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -575,7 +579,7 @@ Start build again. This should become build number 2. ... "build_id": "rome/2", ... "build_number": 2, ... "log": "/logs/rome/2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -596,7 +600,7 @@ Start build again. This should become build number 2. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -630,7 +634,7 @@ Start build again. This should become build number 2. ... "build_id": "rome/2", ... "build_number": 2, ... "log": "/logs/rome/2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -662,7 +666,7 @@ Start build again. This should become build number 2. ... { ... "build_id": "rome/2", ... "action_id": "1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "", @@ -678,7 +682,7 @@ Start build again. This should become build number 2. ... "build_id": "rome/2", ... "build_number": 2, ... "log": "/logs/rome/2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -694,7 +698,7 @@ Start build again. This should become build number 2. ... { ... "build_id": "rome/2", ... "action_id": "2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "hey ho", @@ -710,7 +714,7 @@ Start build again. This should become build number 2. ... { ... "build_id": "rome/2", ... "action_id": "3", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "hey ho", @@ -725,7 +729,7 @@ Start build again. This should become build number 2. ... { ... "build_id": "rome/2", ... "build_number": 2, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -741,7 +745,7 @@ Start build again. This should become build number 2. ... { ... "build_id": "rome/2", ... "action_id": "4", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "", @@ -763,7 +767,7 @@ Start build again. This should become build number 2. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -797,7 +801,7 @@ Start build again. This should become build number 2. ... "build_id": "rome/2", ... "build_number": 2, ... "log": "/logs/rome/2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": { ... "foo": "bar" @@ -830,6 +834,29 @@ Start build again. This should become build number 2. ... ] ... } + + WHEN user makes request DELETE /projects/bad_rome + AND user makes request DELETE /projects/rome + AND user makes request DELETE /projects/constantinople + AND user makes request DELETE /pipelines/construct + AND user makes request DELETE /workers/${OBELIX} + AND user makes request DELETE /builds/rome/1 + AND user makes request DELETE /builds/rome/2 + AND user makes request DELETE /logs/rome/1 + AND user makes request DELETE /logs/rome/2 + + WHEN user makes request GET /projects + THEN body matches {"projects":[]} + + WHEN user makes request GET /pipelines + THEN body matches {"pipelines":[]} + + WHEN user makes request GET /builds + THEN body matches {"builds":[]} + + WHEN user makes request GET /logs + THEN body matches {"log":[]} + FINALLY stop ick controller @@ -841,20 +868,23 @@ This scenario tests the controller API to simulate a build. Set up the controller. - GIVEN an RSA key pair for token signing - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify - AND an access token for user with scopes + GIVEN an access token for user with scopes + ... uapi_pipelines_get ... uapi_pipelines_post + ... uapi_pipelines_id_delete + ... uapi_projects_get ... uapi_projects_post + ... uapi_projects_id_delete ... uapi_projects_id_status_put ... uapi_projects_id_status_get ... uapi_projects_id_builds_get ... uapi_workers_id_get + ... uapi_workers_id_delete ... uapi_builds_get + ... uapi_builds_id_delete ... uapi_builds_id_get + ... uapi_logs_get + ... uapi_logs_id_delete ... uapi_logs_id_get AND a running ick controller @@ -891,6 +921,7 @@ Register a worker. ... { ... } THEN result has status code 201 + AND worker id is OBELIX Build the first project. @@ -909,7 +940,7 @@ Build the first project. ... "build_id": "first/1", ... "action_id": "1", ... "build_number": 1, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "first", ... "exit_code": 0, ... "stdout": "", @@ -930,7 +961,7 @@ Build the first project. ... "build_id": "first/1", ... "action_id": "2", ... "build_number": 1, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "first", ... "exit_code": 0, ... "stdout": "", @@ -950,7 +981,7 @@ Build the first project. ... "build_id": "first/1", ... "action_id": "3", ... "build_number": 1, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "first", ... "exit_code": 0, ... "stdout": "", @@ -981,7 +1012,7 @@ Build second project. ... { ... "build_id": "second/1", ... "action_id": "1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "second", ... "exit_code": 0, ... "stdout": "", @@ -1001,7 +1032,7 @@ Build second project. ... { ... "build_id": "second/1", ... "action_id": "2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "second", ... "exit_code": 0, ... "stdout": "", @@ -1021,7 +1052,7 @@ Build second project. ... "build_id": "second/1", ... "action_id": "3", ... "build_number": 1, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "second", ... "exit_code": 0, ... "stdout": "", @@ -1038,8 +1069,29 @@ Build second project. Finish up. - FINALLY stop ick controller + WHEN user makes request DELETE /projects/first + AND user makes request DELETE /projects/second + AND user makes request DELETE /pipelines/do_something + AND user makes request DELETE /workers/${OBELIX} + AND user makes request DELETE /builds/first/1 + AND user makes request DELETE /builds/second/1 + AND user makes request DELETE /logs/first/1 + AND user makes request DELETE /logs/second/1 + + WHEN user makes request GET /projects + THEN body matches {"projects":[]} + + WHEN user makes request GET /pipelines + THEN body matches {"pipelines":[]} + + WHEN user makes request GET /builds + THEN body matches {"builds":[]} + + WHEN user makes request GET /logs + THEN body matches {"log":[]} + + FINALLY stop ick controller # Build two projects concurrently @@ -1049,20 +1101,23 @@ This scenario tests the controller API to simulate a build. Set up the controller. - GIVEN an RSA key pair for token signing - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify - AND an access token for user with scopes + GIVEN an access token for user with scopes + ... uapi_pipelines_get ... uapi_pipelines_post + ... uapi_pipelines_id_delete + ... uapi_projects_get ... uapi_projects_post + ... uapi_projects_id_delete ... uapi_projects_id_status_put ... uapi_projects_id_status_get ... uapi_projects_id_builds_get ... uapi_workers_id_get + ... uapi_workers_id_delete ... uapi_builds_get + ... uapi_builds_id_delete ... uapi_builds_id_get + ... uapi_logs_get + ... uapi_logs_id_delete ... uapi_logs_id_get AND a running ick controller @@ -1101,6 +1156,7 @@ Register a couple of workers. ... { ... } THEN result has status code 201 + AND worker id is ASTERIX GIVEN an access token for obelix with scopes ... uapi_workers_post @@ -1110,6 +1166,7 @@ Register a couple of workers. ... { ... } THEN result has status code 201 + AND worker id is OBELIX Trigger both projects. @@ -1147,7 +1204,7 @@ Trigger both projects. ... "build_id": "first/1", ... "action_id": "1", ... "build_number": 1, - ... "worker": "asterix", + ... "worker": "${ASTERIX}", ... "project": "first", ... "exit_code": 0, ... "stdout": "", @@ -1175,7 +1232,7 @@ Trigger both projects. ... "build_id": "second/1", ... "action_id": "1", ... "build_number": 1, - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "second", ... "exit_code": 0, ... "stdout": "", @@ -1195,7 +1252,7 @@ Trigger both projects. ... { ... "build_id": "first/1", ... "action_id": "2", - ... "worker": "asterix", + ... "worker": "${ASTERIX}", ... "project": "first", ... "exit_code": 0, ... "stdout": "", @@ -1214,7 +1271,7 @@ Trigger both projects. ... { ... "build_id": "first/1", ... "action_id": "3", - ... "worker": "asterix", + ... "worker": "${ASTERIX}", ... "project": "first", ... "exit_code": 0, ... "stdout": "", @@ -1230,7 +1287,7 @@ Trigger both projects. ... { ... "build_id": "second/1", ... "action_id": "2", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "second", ... "exit_code": 0, ... "stdout": "", @@ -1249,7 +1306,7 @@ Trigger both projects. ... { ... "build_id": "second/1", ... "action_id": "3", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "second", ... "exit_code": 0, ... "stdout": "", @@ -1264,6 +1321,28 @@ Trigger both projects. WHEN user requests list of builds THEN the list of builds is ["first/1", "second/1"] -Finish up. +Finish up. Delete the resource we created. + + WHEN user makes request DELETE /projects/first + AND user makes request DELETE /projects/second + AND user makes request DELETE /pipelines/do_something + AND user makes request DELETE /workers/${ASTERIX} + AND user makes request DELETE /workers/${OBELIX} + AND user makes request DELETE /builds/first/1 + AND user makes request DELETE /builds/second/1 + AND user makes request DELETE /logs/first/1 + AND user makes request DELETE /logs/second/1 + + WHEN user makes request GET /projects + THEN body matches {"projects":[]} + + WHEN user makes request GET /pipelines + THEN body matches {"pipelines":[]} + + WHEN user makes request GET /builds + THEN body matches {"builds":[]} + + WHEN user makes request GET /logs + THEN body matches {"log":[]} FINALLY stop ick controller diff --git a/yarns/500-build-fail.yarn b/yarns/500-build-fail.yarn index 3373c2f..6ca06c6 100644 --- a/yarns/500-build-fail.yarn +++ b/yarns/500-build-fail.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -26,20 +26,24 @@ build step fails. Set up the controller. - GIVEN an RSA key pair for token signing - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify - AND an access token for user with scopes + GIVEN an access token for user with scopes + ... uapi_pipelines_get + ... uapi_pipelines_id_delete ... uapi_pipelines_post + ... uapi_projects_get + ... uapi_projects_id_delete ... uapi_projects_post ... uapi_projects_id_status_put ... uapi_projects_id_status_get ... uapi_projects_id_builds_get + ... uapi_workers_get + ... uapi_workers_id_delete ... uapi_workers_id_get ... uapi_builds_get + ... uapi_builds_id_delete ... uapi_builds_id_get + ... uapi_logs_get + ... uapi_logs_id_delete ... uapi_logs_id_get AND a running ick controller @@ -71,6 +75,7 @@ Register a worker. ... { ... } THEN result has status code 201 + AND worker id is OBELIX Trigger build. @@ -86,7 +91,7 @@ Worker wants work and gets the first step to run. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": {}, ... "action_id": "1", @@ -103,7 +108,7 @@ failure. ... { ... "build_id": "rome/1", ... "action_id": "1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 1, ... "stdout": "", @@ -121,7 +126,7 @@ Worker is next told to notify end of build. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": {}, ... "action_id": "4", @@ -134,7 +139,7 @@ Worker is next told to notify end of build. ... { ... "build_id": "rome/1", ... "action_id": "4", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "exit_code": 0, ... "stdout": "", @@ -151,11 +156,11 @@ The build has ended, and there's no more work to do. User sees changed status. - WHEN user makes request GET /workers/obelix + WHEN user makes request GET /workers/${OBELIX} THEN result has status code 200 AND body matches ... { - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "doing": {} ... } @@ -170,7 +175,7 @@ There's a build with a log. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": {}, ... "graph": { @@ -211,7 +216,7 @@ There's a build with a log. ... "build_id": "rome/1", ... "build_number": 1, ... "log": "/logs/rome/1", - ... "worker": "obelix", + ... "worker": "${OBELIX}", ... "project": "rome", ... "parameters": {}, ... "graph": { @@ -248,4 +253,22 @@ There's a build with a log. AND result has header Content-Type: text/plain AND body text contains "eek!" + WHEN user makes request DELETE /projects/rome + AND user makes request DELETE /pipelines/construct + AND user makes request DELETE /workers/${OBELIX} + AND user makes request DELETE /builds/rome/1 + AND user makes request DELETE /logs/rome/1 + + WHEN user makes request GET /projects + THEN body matches {"projects":[]} + + WHEN user makes request GET /pipelines + THEN body matches {"pipelines":[]} + + WHEN user makes request GET /builds + THEN body matches {"builds":[]} + + WHEN user makes request GET /logs + THEN body matches {"log":[]} + FINALLY stop ick controller diff --git a/yarns/600-unauthz.yarn b/yarns/600-unauthz.yarn index 1c928ac..ab33404 100644 --- a/yarns/600-unauthz.yarn +++ b/yarns/600-unauthz.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -26,12 +26,9 @@ returned. Set up the controller. - GIVEN an RSA key pair for token signing - AND controller config uses statedir at the state directory - AND controller config uses https://blobs.example.com as artifact store - AND controller config uses https://auth.example.com as authentication - AND controller config uses https://notify.example.com as notify - AND an access token for user with scopes + GIVEN an access token for user with scopes + ... uapi_projects_get + ... uapi_projects_id_delete ... uapi_projects_post ... uapi_projects_id_status_put ... uapi_projects_id_status_get @@ -88,4 +85,8 @@ Set up the controller. WHEN outsider makes request POST /work with an invalid token and body {} THEN result has status code 401 + WHEN user makes request DELETE /projects/rome + WHEN user makes request GET /projects + THEN body matches {"projects":[]} + FINALLY stop ick controller diff --git a/yarns/700-artifact-store.yarn b/yarns/700-artifact-store.yarn index 2dcea2e..cadc83c 100644 --- a/yarns/700-artifact-store.yarn +++ b/yarns/700-artifact-store.yarn @@ -1,6 +1,6 @@ <!-- -Copyright 2017-2018 Lars Wirzenius +Copyright 2017-2019 Lars Wirzenius This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by @@ -27,18 +27,23 @@ simple, in fact, it will certainly change in the future. Set up the artifact store. - GIVEN an RSA key pair for token signing - AND artifact store config uses blobs at the blob directory - AND an access token for user with scopes + GIVEN an access token for user with scopes + ... uapi_blobs_id_delete ... uapi_blobs_id_put ... uapi_blobs_id_get - AND a running artifact store + AND a running ick controller + +<!-- + +FIXME: This is disabled, until the artifact store supports deletion. Try to get a non-existent blob. It should result in an error. WHEN user retrieves /blobs/cake from artifact store THEN result has status code 404 +--> + Create and store a blob, retrieve it and verify we get it back intack. WHEN user creates a blob named cake with random data @@ -49,4 +54,16 @@ Create and store a blob, retrieve it and verify we get it back intack. THEN result has status code 200 AND body is the same as the blob cake - FINALLY stop artifact store +<!-- + +FIXME: This is disabled, until the artifact store supports deletion. + +Delete the cake. + + WHEN user deletes /blob/cake from artifact store + WHEN user retrieves /blobs/cake from artifact store + THEN result has status code 404 + +--> + + FINALLY stop ick controller diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn index 4dcbd8b..92acaa4 100644 --- a/yarns/900-implements.yarn +++ b/yarns/900-implements.yarn @@ -32,6 +32,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. IMPLEMENTS WHEN (\S+) makes request GET (\S+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) token = get_token(user) url = V['url'] http(V, get, url + path, token=token) @@ -40,8 +41,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. user = get_next_match() path = get_next_match() token = get_token(user) - url = V['bsurl'] - http(V, get_blob, url + path, token=token) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, get_blob, asurl + path, token=token) + + IMPLEMENTS WHEN (\S+) deletes (\S+) from artifact store + user = get_next_match() + path = get_next_match() + token = get_token(user) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, delete, asurl + path, token=token) IMPLEMENTS WHEN (\S+) makes request GET (\S+) with an invalid token user = get_next_match() @@ -54,6 +66,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. user = get_next_match() path = get_next_match() body = get_next_match() + body = expand_vars(body, V) + V['xxxPOSTbodyvalid'] = body token = get_token(user) url = V['url'] http(V, post, url + path, body=body, token=token) @@ -62,6 +76,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. user = get_next_match() path = get_next_match() body = get_next_match() + body = expand_vars(body, V) + V['xxxPOSTbody'] = body token = get_token(user) url = V['url'] http(V, post, url + path, body=body, token='invalid') @@ -69,7 +85,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with a valid token and body (.+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) body = get_next_match() + body = expand_vars(body, V) + V['xxxPUTbody'] = body token = get_token(user) url = V['url'] http(V, put, url + path, body=body, token=token) @@ -80,12 +99,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. path = get_next_match() body = cat(filename) token = get_token(user) - url = V['bsurl'] - http(V, put_blob, url + path, body=body, token=token) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, put_blob, asurl + path, body=body, token=token) IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with an invalid token user = get_next_match() path = get_next_match() + path = expand_vars(path, V) body = '{}' token = get_token(user) url = V['url'] @@ -94,18 +116,25 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. IMPLEMENTS WHEN (\S+) makes request DELETE (\S+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) token = get_token(user) url = V['url'] http(V, delete, url + path, token=token) ## HTTP response inspection + IMPLEMENTS THEN worker id is (\S+) + varname = get_next_match() + body = json.loads(V['body']) + V[varname] = body['worker'] + IMPLEMENTS THEN result has status code (\d+) expected = int(get_next_match()) assertEqual(expected, V['status_code']) IMPLEMENTS THEN body matches (.+) expected_text = get_next_match() + expected_text = expand_vars(expected_text, V) expected = json.loads(expected_text) actual = json.loads(V['body']) print 'expected' diff --git a/yarns/900-local.yarn b/yarns/900-local.yarn deleted file mode 100644 index e9e8a7c..0000000 --- a/yarns/900-local.yarn +++ /dev/null @@ -1,121 +0,0 @@ -<!-- - -Copyright 2017-2019 Lars Wirzenius - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see <http://www.gnu.org/licenses/>. - ---> - -# Scenario step implementations for locally managed ick - -## Authentication setup - - IMPLEMENTS GIVEN an RSA key pair for token signing - argv = [ - os.path.join(srcdir, 'generate-rsa-key'), - 'token.key', - ] - cliapp.runcmd(argv, stdout=None, stderr=None) - - IMPLEMENTS GIVEN an access token for (\S+) with scopes (.+) - user = get_next_match() - scopes = get_next_match() - key = open('token.key').read() - argv = [ - os.path.join(srcdir, 'create-token'), - scopes, - user, - ] - token = cliapp.runcmd(argv, feed_stdin=key) - store_token(user, token) - V['issuer'] = 'localhost' - V['audience'] = user - -## Controller configuration - - IMPLEMENTS GIVEN controller config uses (\S+) at the state directory - V['statedir'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as artifact store - V['artifact_store'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as authentication - V['auth_url'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as notify - V['notify_url'] = get_next_match() - assert V['notify_url'] is not None - -## Start and stop the controller - - IMPLEMENTS GIVEN a running ick controller - start_controller() - - IMPLEMENTS WHEN user stops ick controller - stop_controller() - - IMPLEMENTS FINALLY stop ick controller - stop_controller() - -## Controller state inspection - - IMPLEMENTS THEN controller state directory contains project (\S+) - name = get_next_match() - basename = encode_basename(name) - filename = os.path.join(V['statedir'], 'projects', basename) - print 'name', name - print 'basename', basename - print 'filename', filename - assertTrue(os.path.exists(filename)) - - IMPLEMENTS THEN controller state directory contains worker (\S+) - name = get_next_match() - basename = encode_basename(name) - filename = os.path.join(V['statedir'], 'workers', basename) - print 'filename', filename - assertTrue(os.path.exists(filename)) - -## Check version result - - IMPLEMENTS THEN artifact store URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['artifact_store'] - assertEqual(actual, expected) - - IMPLEMENTS THEN authentication URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['auth_url'] - assertEqual(actual, expected) - - IMPLEMENTS THEN notify URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['notify_url'] - assertEqual(actual, expected) - -## Start and stop artifact store - - IMPLEMENTS GIVEN artifact store config uses (\S+) at the blob directory - V['blobdir'] = get_next_match() - - IMPLEMENTS GIVEN a running artifact store - start_artifact_store() - - IMPLEMENTS FINALLY stop artifact store - stop_artifact_store() diff --git a/yarns/900-remote.yarn b/yarns/900-remote.yarn index 3c0443c..5e84c13 100644 --- a/yarns/900-remote.yarn +++ b/yarns/900-remote.yarn @@ -21,22 +21,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. ## Authentication setup - IMPLEMENTS GIVEN an RSA key pair for token signing - V['private_key_file'] = os.environ['ICK_PRIVATE_KEY'] - assertTrue(os.path.exists(V['private_key_file'])) - IMPLEMENTS GIVEN an access token for (\S+) with scopes (.+) user = get_next_match() - scopes = get_next_match() - key = open(V['private_key_file']).read() - argv = [ - os.path.join(srcdir, 'create-token'), - scopes, - ] - token = cliapp.runcmd(argv, feed_stdin=key) + scopes = get_next_match().split() + create_api_client(user, scopes) + token = get_api_token(user, scopes) store_token(user, token) - V['issuer'] = 'localhost' - V['audience'] = 'localhost' ## Controller configuration @@ -46,13 +36,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. ## Start and stop the controller IMPLEMENTS GIVEN a running ick controller - V['url'] = os.environ['ICK_URL'] + V['url'] = os.environ['CONTROLLER'] IMPLEMENTS WHEN user stops ick controller pass IMPLEMENTS FINALLY stop ick controller - pass + for client_id in get_client_ids(): + delete_api_client(client_id) ## Controller state inspection diff --git a/yarns/lib.py b/yarns/lib.py index 5fbd5ab..6d8f2cf 100644 --- a/yarns/lib.py +++ b/yarns/lib.py @@ -19,11 +19,13 @@ import errno import json import os import random +import re import signal import socket import sys import time import urllib +import uuid import cliapp import requests @@ -37,119 +39,80 @@ datadir = os.environ['DATADIR'] V = Variables(datadir) -def start_controller(): - port = V['port'] = random_free_port() - - V['url'] = 'http://127.0.0.1:{}'.format(V['port']) - - filename = 'ick_controller.yaml' - env = dict(os.environ) - env['ICK_CONTROLLER_CONFIG'] = filename - write_yaml(filename, { - 'token-issuer': V['issuer'], - 'token-audience': V['audience'], - 'token-public-key': cat('token.key.pub'), - 'log': [ - { - 'filename': 'ick_controller.log', - }, - ], - 'statedir': V['statedir'], - 'apt-server': 'localhost', - 'artifact-store': V['artifact_store'], - 'auth-url': V['auth_url'], - 'notify-url': V['notify_url'], - }) - - V['pid'] = gunicorn('ick_controller', 'app', port, env) - - -def stop_controller(): - if V['pid'] is not None: - os.kill(int(V['pid']), signal.SIGTERM) - - -def start_artifact_store(): - port = V['bsport'] = random_free_port() - - V['bsurl'] = 'http://127.0.0.1:{}'.format(V['bsport']) - - filename = 'artifact_store.yaml' - env = dict(os.environ) - env['ARTIFACT_STORE_CONFIG'] = filename - write_yaml(filename, { - 'token-issuer': V['issuer'], - 'token-audience': V['audience'], - 'token-public-key': cat('token.key.pub'), - 'log': [ - { - 'filename': 'artifact_store.log', - }, - ], - 'blobdir': V['blobdir'], - }) - - V['bspid'] = gunicorn('artifact_store', 'app', port, env) - - -def stop_artifact_store(): - if V['pid'] is not None: - os.kill(int(V['bspid']), signal.SIGTERM) - - -def write_yaml(filename, obj): - yaml.safe_dump(obj, open(filename, 'w')) - - -def gunicorn(module_name, var_name, port, env): - log_filename = '{}.gunicorn.log'.format(module_name) - pid_filename = '{}.pid'.format(module_name) - - argv = [ - 'gunicorn3', - '--daemon', - '--bind', '127.0.0.1:{}'.format(port), - '--log-file', log_filename, - '--log-level', 'debug', - '-p', pid_filename, - '{}:{}'.format(module_name, var_name), - ] - cliapp.runcmd(argv, env=env) - wait_for_port(port) - return int(cat(pid_filename)) - - -def random_free_port(): - MAX = 1000 - for i in range(MAX): - port = random.randint(1025, 2**15-1) - s = socket.socket() - try: - s.bind(('0.0.0.0', port)) - except OSError as e: - if e.errno == errno.EADDRINUSE: - continue - print('cannot find a random free port') - raise - s.close() - break - print('picked port', port) - return port - - -def wait_for_port(port): - MAX = 5 - t = time.time() - while time.time() < t + MAX: - try: - s = socket.socket() - s.connect(('127.0.0.1', port)) - except socket.error: - time.sleep(0.1) - except OSError as e: - raise - else: - return +def remember_client_id(alias, client_id, client_secret): + clients = V['clients'] + if clients is None: + clients = {} + clients[alias] = { + 'client_id': client_id, + 'client_secret': client_secret, + } + V['clients'] = clients + + +def get_client_id(alias): + clients = V['clients'] or {} + return clients[alias]['client_id'] + + +def get_client_ids(): + clients = V['clients'] or {} + return [x['client_id'] for x in clients.values()] + + +def get_client_secret(alias): + clients = V['clients'] or {} + return clients[alias]['client_secret'] + + +def create_api_client(alias, scopes): + client_id = str(uuid.uuid4()) + client_secret = str(uuid.uuid4()) + print('invented client id', client_id) + api = os.environ['CONTROLLER'] + print('controller URL', api) + secrets = os.environ['SECRETS'] + print('secrets', secrets) + base_argv = ['qvisqvetool', '--secrets', secrets, '-a', api] + print('base_argv', base_argv) + cliapp.runcmd(base_argv + ['create', 'client', client_id, client_secret]) + cliapp.runcmd(base_argv + ['allow-scope', 'client', client_id] + scopes) + remember_client_id(alias, client_id, client_secret) + + +def delete_api_client(client_id): + api = os.environ['CONTROLLER'] + secrets = os.environ['SECRETS'] + base_argv = ['qvisqvetool', '--secrets', secrets, '-a', api] + cliapp.runcmd(base_argv + ['delete', 'client', client_id]) + + +def get_api_token(alias, scopes): + print('getting token for', alias) + + client_id = get_client_id(alias) + client_secret = get_client_secret(alias) + api = os.environ['CONTROLLER'] + + auth = (client_id, client_secret) + data = { + 'grant_type': 'client_credentials', + 'scope': ' '.join(scopes), + } + + url = '{}/token'.format(api) + + print('url', url) + print('auth', auth) + print('data', data) + r = requests.post(url, auth=auth, data=data) + if not r.ok: + sys.exit('Error getting token: %s %s' % (r.status_code, r.text)) + + token = r.json()['access_token'] + print('token', token) + return token + def unescape(s): t = '' @@ -186,6 +149,12 @@ def get_token(user): def http(V, func, url, **kwargs): + V['request'] = { + 'func': repr(func), + 'url': url, + 'kwargs': kwargs, + } + print('http', func, url, kwargs) status, content_type, headers, body = func(url, **kwargs) V['status_code'] = status V['content_type'] = content_type @@ -201,6 +170,11 @@ def get(url, token): return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text +def get_version(url): + status, ctype, headers, text = get(url + '/version', 'no token') + assert ctype == 'application/json' + return json.loads(text) + def get_blob(url, token): headers = { 'Authorization': 'Bearer {}'.format(token), @@ -309,5 +283,15 @@ def list_diff(a, b): return None -def encode_basename(basename): - return urllib.quote(basename, safe='') +def expand_vars(text, variables): + result = '' + while text: + m = re.search(r'\${(?P<name>[^}]+)}', text) + if not m: + result += text + break + name = m.group('name') + print('expanding ', name) + result += text[:m.start()] + variables[name] + text = text[m.end():] + return result |