From 1fc41ad97411b435c512ba3a0de63929eda9c33d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 28 Jun 2019 08:00:28 +0300 Subject: Change: make yarns run against a remote Ick instance, not local --- check | 72 +++++--------- create-token | 47 ---------- generate-rsa-key | 34 ------- setup.py | 3 +- yarns/000.yarn | 33 +++---- yarns/100-projects.yarn | 9 +- yarns/150-pipelines.yarn | 9 +- yarns/200-version.yarn | 12 +-- yarns/300-workers.yarn | 33 +++---- yarns/400-build.yarn | 209 ++++++++++++++++++++++++++++------------- yarns/500-build-fail.yarn | 53 ++++++++--- yarns/600-unauthz.yarn | 15 +-- yarns/700-artifact-store.yarn | 29 ++++-- yarns/900-implements.yarn | 37 +++++++- yarns/900-local.yarn | 121 ------------------------ yarns/900-remote.yarn | 21 ++--- yarns/lib.py | 214 +++++++++++++++++++----------------------- 17 files changed, 409 insertions(+), 542 deletions(-) delete mode 100755 create-token delete mode 100755 generate-rsa-key delete mode 100644 yarns/900-local.yarn diff --git a/check b/check index 133b9c0..1e98ee1 100755 --- a/check +++ b/check @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2017-2018 Lars Wirzenius +# Copyright 2017-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -30,32 +30,16 @@ title() } -title Remote or local yarns? -remote=no -unit=yes -yarns=yes +title Remote yarns? +yarns=no if [ "$#" -gt 0 ] then case "$1" in https://*) - remote=yes - unit=no yarns=yes remote_url="$1" shift 1 ;; - yarns) - remote=no - unit=no - yarns=yes - shift 1 - ;; - local) - remote=no - unit=yes - yarns=no - shift 1 - ;; *) echo "Don't understand args: $@" 1>&2 exit 1 @@ -64,51 +48,41 @@ then fi -if [ "$unit" = yes ] -then - title Unit tests - python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 +title Unit tests +python3 -m CoverageTestRunner --ignore-missing-from=without-tests ick2 - if [ -e .git ] - then - sources="$(git ls-files | grep -Fvxf copyright-exceptions)" +if [ -e .git ] +then + sources="$(git ls-files | grep -Fvxf copyright-exceptions)" - title Copyright statements - copyright-statement-lint $sources + title Copyright statements + copyright-statement-lint $sources - title Copyright licences - ./is-agpl3+ $sources - fi + title Copyright licences + ./is-agpl3+ $sources +fi - python_sources="ick_controller.py worker_manager ick2 icktool" +python_sources="ick_controller.py worker_manager ick2 icktool" - title pycodestyle - pycodestyle ick2 $python_sources +title pycodestyle +pycodestyle ick2 $python_sources - if command -v pylint3 > /dev/null - then - title pylint3 - pylint3 --rcfile pylint.conf $python_sources - fi +if command -v pylint3 > /dev/null +then + title pylint3 + pylint3 --rcfile pylint.conf $python_sources fi if [ "$yarns" = yes ] then title Yarns - if [ "$remote" = no ] - then - impl=yarns/900-local.yarn - args="" - else - impl=yarns/900-remote.yarn - args="--env ICK_URL=$remote_url" - fi - yarn yarns/[^9]*.yarn yarns/900-implements.yarn "$impl" \ + yarn yarns/*.yarn \ --shell python2 \ --shell-arg '' \ --shell-library yarns/lib.py \ --cd-datadir \ - $args \ + --env "CONTROLLER=$remote_url" \ + --env "SECRETS=$HOME/.config/qvarn/createtoken.conf" \ "$@" fi diff --git a/create-token b/create-token deleted file mode 100755 index 55a7f7e..0000000 --- a/create-token +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/python3 -# Copyright (C) 2017-2018 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - - -import sys -import time - -import Crypto.PublicKey.RSA - -import apifw - - -# FIXME: These should agree with how ick controller is configured. -# See the Ansible playbook. -iss = 'localhost' - - -key_text = sys.stdin.read() -key = Crypto.PublicKey.RSA.importKey(key_text) - -scopes = ' '.join(sys.argv[1].split()) -aud = sys.argv[2] - -now = time.time() -claims = { - 'iss': iss, - 'sub': 'subject-uuid', - 'aud': aud, - 'exp': now + 86400, # FIXME: This is silly long - 'scope': scopes, -} - -token = apifw.create_token(claims, key) -sys.stdout.write(token.decode('ascii')) diff --git a/generate-rsa-key b/generate-rsa-key deleted file mode 100755 index e44a796..0000000 --- a/generate-rsa-key +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/python3 -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - - -import sys - -import Crypto.PublicKey.RSA - - -RSA_KEY_BITS = 4096 # A nice, currently safe length - -key = Crypto.PublicKey.RSA.generate(RSA_KEY_BITS) - -filename = sys.argv[1] - -def write(filename, byts): - with open(filename, 'w') as f: - f.write(byts.decode('ascii')) - -write(filename, key.exportKey('PEM')) -write(filename + '.pub', key.exportKey('OpenSSH')) diff --git a/setup.py b/setup.py index 39a7977..fc4d70b 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright (C) 2017-2018 Lars Wirzenius +# Copyright (C) 2017-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -53,7 +53,6 @@ setup( ], packages=['ick2'], scripts=[ - 'create-token', 'start_ick', 'start_artifact_store', 'start_notification_service', diff --git a/yarns/000.yarn b/yarns/000.yarn index d5db1a1..12bdea2 100644 --- a/yarns/000.yarn +++ b/yarns/000.yarn @@ -1,6 +1,6 @@ + Create and store a blob, retrieve it and verify we get it back intack. WHEN user creates a blob named cake with random data @@ -49,4 +54,16 @@ Create and store a blob, retrieve it and verify we get it back intack. THEN result has status code 200 AND body is the same as the blob cake - FINALLY stop artifact store + + + FINALLY stop ick controller diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn index 4dcbd8b..92acaa4 100644 --- a/yarns/900-implements.yarn +++ b/yarns/900-implements.yarn @@ -32,6 +32,7 @@ along with this program. If not, see . IMPLEMENTS WHEN (\S+) makes request GET (\S+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) token = get_token(user) url = V['url'] http(V, get, url + path, token=token) @@ -40,8 +41,19 @@ along with this program. If not, see . user = get_next_match() path = get_next_match() token = get_token(user) - url = V['bsurl'] - http(V, get_blob, url + path, token=token) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, get_blob, asurl + path, token=token) + + IMPLEMENTS WHEN (\S+) deletes (\S+) from artifact store + user = get_next_match() + path = get_next_match() + token = get_token(user) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, delete, asurl + path, token=token) IMPLEMENTS WHEN (\S+) makes request GET (\S+) with an invalid token user = get_next_match() @@ -54,6 +66,8 @@ along with this program. If not, see . user = get_next_match() path = get_next_match() body = get_next_match() + body = expand_vars(body, V) + V['xxxPOSTbodyvalid'] = body token = get_token(user) url = V['url'] http(V, post, url + path, body=body, token=token) @@ -62,6 +76,8 @@ along with this program. If not, see . user = get_next_match() path = get_next_match() body = get_next_match() + body = expand_vars(body, V) + V['xxxPOSTbody'] = body token = get_token(user) url = V['url'] http(V, post, url + path, body=body, token='invalid') @@ -69,7 +85,10 @@ along with this program. If not, see . IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with a valid token and body (.+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) body = get_next_match() + body = expand_vars(body, V) + V['xxxPUTbody'] = body token = get_token(user) url = V['url'] http(V, put, url + path, body=body, token=token) @@ -80,12 +99,15 @@ along with this program. If not, see . path = get_next_match() body = cat(filename) token = get_token(user) - url = V['bsurl'] - http(V, put_blob, url + path, body=body, token=token) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, put_blob, asurl + path, body=body, token=token) IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with an invalid token user = get_next_match() path = get_next_match() + path = expand_vars(path, V) body = '{}' token = get_token(user) url = V['url'] @@ -94,18 +116,25 @@ along with this program. If not, see . IMPLEMENTS WHEN (\S+) makes request DELETE (\S+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) token = get_token(user) url = V['url'] http(V, delete, url + path, token=token) ## HTTP response inspection + IMPLEMENTS THEN worker id is (\S+) + varname = get_next_match() + body = json.loads(V['body']) + V[varname] = body['worker'] + IMPLEMENTS THEN result has status code (\d+) expected = int(get_next_match()) assertEqual(expected, V['status_code']) IMPLEMENTS THEN body matches (.+) expected_text = get_next_match() + expected_text = expand_vars(expected_text, V) expected = json.loads(expected_text) actual = json.loads(V['body']) print 'expected' diff --git a/yarns/900-local.yarn b/yarns/900-local.yarn deleted file mode 100644 index e9e8a7c..0000000 --- a/yarns/900-local.yarn +++ /dev/null @@ -1,121 +0,0 @@ - - -# Scenario step implementations for locally managed ick - -## Authentication setup - - IMPLEMENTS GIVEN an RSA key pair for token signing - argv = [ - os.path.join(srcdir, 'generate-rsa-key'), - 'token.key', - ] - cliapp.runcmd(argv, stdout=None, stderr=None) - - IMPLEMENTS GIVEN an access token for (\S+) with scopes (.+) - user = get_next_match() - scopes = get_next_match() - key = open('token.key').read() - argv = [ - os.path.join(srcdir, 'create-token'), - scopes, - user, - ] - token = cliapp.runcmd(argv, feed_stdin=key) - store_token(user, token) - V['issuer'] = 'localhost' - V['audience'] = user - -## Controller configuration - - IMPLEMENTS GIVEN controller config uses (\S+) at the state directory - V['statedir'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as artifact store - V['artifact_store'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as authentication - V['auth_url'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as notify - V['notify_url'] = get_next_match() - assert V['notify_url'] is not None - -## Start and stop the controller - - IMPLEMENTS GIVEN a running ick controller - start_controller() - - IMPLEMENTS WHEN user stops ick controller - stop_controller() - - IMPLEMENTS FINALLY stop ick controller - stop_controller() - -## Controller state inspection - - IMPLEMENTS THEN controller state directory contains project (\S+) - name = get_next_match() - basename = encode_basename(name) - filename = os.path.join(V['statedir'], 'projects', basename) - print 'name', name - print 'basename', basename - print 'filename', filename - assertTrue(os.path.exists(filename)) - - IMPLEMENTS THEN controller state directory contains worker (\S+) - name = get_next_match() - basename = encode_basename(name) - filename = os.path.join(V['statedir'], 'workers', basename) - print 'filename', filename - assertTrue(os.path.exists(filename)) - -## Check version result - - IMPLEMENTS THEN artifact store URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['artifact_store'] - assertEqual(actual, expected) - - IMPLEMENTS THEN authentication URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['auth_url'] - assertEqual(actual, expected) - - IMPLEMENTS THEN notify URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['notify_url'] - assertEqual(actual, expected) - -## Start and stop artifact store - - IMPLEMENTS GIVEN artifact store config uses (\S+) at the blob directory - V['blobdir'] = get_next_match() - - IMPLEMENTS GIVEN a running artifact store - start_artifact_store() - - IMPLEMENTS FINALLY stop artifact store - stop_artifact_store() diff --git a/yarns/900-remote.yarn b/yarns/900-remote.yarn index 3c0443c..5e84c13 100644 --- a/yarns/900-remote.yarn +++ b/yarns/900-remote.yarn @@ -21,22 +21,12 @@ along with this program. If not, see . ## Authentication setup - IMPLEMENTS GIVEN an RSA key pair for token signing - V['private_key_file'] = os.environ['ICK_PRIVATE_KEY'] - assertTrue(os.path.exists(V['private_key_file'])) - IMPLEMENTS GIVEN an access token for (\S+) with scopes (.+) user = get_next_match() - scopes = get_next_match() - key = open(V['private_key_file']).read() - argv = [ - os.path.join(srcdir, 'create-token'), - scopes, - ] - token = cliapp.runcmd(argv, feed_stdin=key) + scopes = get_next_match().split() + create_api_client(user, scopes) + token = get_api_token(user, scopes) store_token(user, token) - V['issuer'] = 'localhost' - V['audience'] = 'localhost' ## Controller configuration @@ -46,13 +36,14 @@ along with this program. If not, see . ## Start and stop the controller IMPLEMENTS GIVEN a running ick controller - V['url'] = os.environ['ICK_URL'] + V['url'] = os.environ['CONTROLLER'] IMPLEMENTS WHEN user stops ick controller pass IMPLEMENTS FINALLY stop ick controller - pass + for client_id in get_client_ids(): + delete_api_client(client_id) ## Controller state inspection diff --git a/yarns/lib.py b/yarns/lib.py index 5fbd5ab..6d8f2cf 100644 --- a/yarns/lib.py +++ b/yarns/lib.py @@ -19,11 +19,13 @@ import errno import json import os import random +import re import signal import socket import sys import time import urllib +import uuid import cliapp import requests @@ -37,119 +39,80 @@ datadir = os.environ['DATADIR'] V = Variables(datadir) -def start_controller(): - port = V['port'] = random_free_port() - - V['url'] = 'http://127.0.0.1:{}'.format(V['port']) - - filename = 'ick_controller.yaml' - env = dict(os.environ) - env['ICK_CONTROLLER_CONFIG'] = filename - write_yaml(filename, { - 'token-issuer': V['issuer'], - 'token-audience': V['audience'], - 'token-public-key': cat('token.key.pub'), - 'log': [ - { - 'filename': 'ick_controller.log', - }, - ], - 'statedir': V['statedir'], - 'apt-server': 'localhost', - 'artifact-store': V['artifact_store'], - 'auth-url': V['auth_url'], - 'notify-url': V['notify_url'], - }) - - V['pid'] = gunicorn('ick_controller', 'app', port, env) - - -def stop_controller(): - if V['pid'] is not None: - os.kill(int(V['pid']), signal.SIGTERM) - - -def start_artifact_store(): - port = V['bsport'] = random_free_port() - - V['bsurl'] = 'http://127.0.0.1:{}'.format(V['bsport']) - - filename = 'artifact_store.yaml' - env = dict(os.environ) - env['ARTIFACT_STORE_CONFIG'] = filename - write_yaml(filename, { - 'token-issuer': V['issuer'], - 'token-audience': V['audience'], - 'token-public-key': cat('token.key.pub'), - 'log': [ - { - 'filename': 'artifact_store.log', - }, - ], - 'blobdir': V['blobdir'], - }) - - V['bspid'] = gunicorn('artifact_store', 'app', port, env) - - -def stop_artifact_store(): - if V['pid'] is not None: - os.kill(int(V['bspid']), signal.SIGTERM) - - -def write_yaml(filename, obj): - yaml.safe_dump(obj, open(filename, 'w')) - - -def gunicorn(module_name, var_name, port, env): - log_filename = '{}.gunicorn.log'.format(module_name) - pid_filename = '{}.pid'.format(module_name) - - argv = [ - 'gunicorn3', - '--daemon', - '--bind', '127.0.0.1:{}'.format(port), - '--log-file', log_filename, - '--log-level', 'debug', - '-p', pid_filename, - '{}:{}'.format(module_name, var_name), - ] - cliapp.runcmd(argv, env=env) - wait_for_port(port) - return int(cat(pid_filename)) - - -def random_free_port(): - MAX = 1000 - for i in range(MAX): - port = random.randint(1025, 2**15-1) - s = socket.socket() - try: - s.bind(('0.0.0.0', port)) - except OSError as e: - if e.errno == errno.EADDRINUSE: - continue - print('cannot find a random free port') - raise - s.close() - break - print('picked port', port) - return port - - -def wait_for_port(port): - MAX = 5 - t = time.time() - while time.time() < t + MAX: - try: - s = socket.socket() - s.connect(('127.0.0.1', port)) - except socket.error: - time.sleep(0.1) - except OSError as e: - raise - else: - return +def remember_client_id(alias, client_id, client_secret): + clients = V['clients'] + if clients is None: + clients = {} + clients[alias] = { + 'client_id': client_id, + 'client_secret': client_secret, + } + V['clients'] = clients + + +def get_client_id(alias): + clients = V['clients'] or {} + return clients[alias]['client_id'] + + +def get_client_ids(): + clients = V['clients'] or {} + return [x['client_id'] for x in clients.values()] + + +def get_client_secret(alias): + clients = V['clients'] or {} + return clients[alias]['client_secret'] + + +def create_api_client(alias, scopes): + client_id = str(uuid.uuid4()) + client_secret = str(uuid.uuid4()) + print('invented client id', client_id) + api = os.environ['CONTROLLER'] + print('controller URL', api) + secrets = os.environ['SECRETS'] + print('secrets', secrets) + base_argv = ['qvisqvetool', '--secrets', secrets, '-a', api] + print('base_argv', base_argv) + cliapp.runcmd(base_argv + ['create', 'client', client_id, client_secret]) + cliapp.runcmd(base_argv + ['allow-scope', 'client', client_id] + scopes) + remember_client_id(alias, client_id, client_secret) + + +def delete_api_client(client_id): + api = os.environ['CONTROLLER'] + secrets = os.environ['SECRETS'] + base_argv = ['qvisqvetool', '--secrets', secrets, '-a', api] + cliapp.runcmd(base_argv + ['delete', 'client', client_id]) + + +def get_api_token(alias, scopes): + print('getting token for', alias) + + client_id = get_client_id(alias) + client_secret = get_client_secret(alias) + api = os.environ['CONTROLLER'] + + auth = (client_id, client_secret) + data = { + 'grant_type': 'client_credentials', + 'scope': ' '.join(scopes), + } + + url = '{}/token'.format(api) + + print('url', url) + print('auth', auth) + print('data', data) + r = requests.post(url, auth=auth, data=data) + if not r.ok: + sys.exit('Error getting token: %s %s' % (r.status_code, r.text)) + + token = r.json()['access_token'] + print('token', token) + return token + def unescape(s): t = '' @@ -186,6 +149,12 @@ def get_token(user): def http(V, func, url, **kwargs): + V['request'] = { + 'func': repr(func), + 'url': url, + 'kwargs': kwargs, + } + print('http', func, url, kwargs) status, content_type, headers, body = func(url, **kwargs) V['status_code'] = status V['content_type'] = content_type @@ -201,6 +170,11 @@ def get(url, token): return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text +def get_version(url): + status, ctype, headers, text = get(url + '/version', 'no token') + assert ctype == 'application/json' + return json.loads(text) + def get_blob(url, token): headers = { 'Authorization': 'Bearer {}'.format(token), @@ -309,5 +283,15 @@ def list_diff(a, b): return None -def encode_basename(basename): - return urllib.quote(basename, safe='') +def expand_vars(text, variables): + result = '' + while text: + m = re.search(r'\${(?P[^}]+)}', text) + if not m: + result += text + break + name = m.group('name') + print('expanding ', name) + result += text[:m.start()] + variables[name] + text = text[m.end():] + return result -- cgit v1.2.1