From 1fc41ad97411b435c512ba3a0de63929eda9c33d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 28 Jun 2019 08:00:28 +0300 Subject: Change: make yarns run against a remote Ick instance, not local --- yarns/000.yarn | 33 +++---- yarns/100-projects.yarn | 9 +- yarns/150-pipelines.yarn | 9 +- yarns/200-version.yarn | 12 +-- yarns/300-workers.yarn | 33 +++---- yarns/400-build.yarn | 209 ++++++++++++++++++++++++++++------------- yarns/500-build-fail.yarn | 53 ++++++++--- yarns/600-unauthz.yarn | 15 +-- yarns/700-artifact-store.yarn | 29 ++++-- yarns/900-implements.yarn | 37 +++++++- yarns/900-local.yarn | 121 ------------------------ yarns/900-remote.yarn | 21 ++--- yarns/lib.py | 214 +++++++++++++++++++----------------------- 13 files changed, 385 insertions(+), 410 deletions(-) delete mode 100644 yarns/900-local.yarn (limited to 'yarns') diff --git a/yarns/000.yarn b/yarns/000.yarn index d5db1a1..12bdea2 100644 --- a/yarns/000.yarn +++ b/yarns/000.yarn @@ -1,6 +1,6 @@ + Create and store a blob, retrieve it and verify we get it back intack. WHEN user creates a blob named cake with random data @@ -49,4 +54,16 @@ Create and store a blob, retrieve it and verify we get it back intack. THEN result has status code 200 AND body is the same as the blob cake - FINALLY stop artifact store + + + FINALLY stop ick controller diff --git a/yarns/900-implements.yarn b/yarns/900-implements.yarn index 4dcbd8b..92acaa4 100644 --- a/yarns/900-implements.yarn +++ b/yarns/900-implements.yarn @@ -32,6 +32,7 @@ along with this program. If not, see . IMPLEMENTS WHEN (\S+) makes request GET (\S+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) token = get_token(user) url = V['url'] http(V, get, url + path, token=token) @@ -40,8 +41,19 @@ along with this program. If not, see . user = get_next_match() path = get_next_match() token = get_token(user) - url = V['bsurl'] - http(V, get_blob, url + path, token=token) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, get_blob, asurl + path, token=token) + + IMPLEMENTS WHEN (\S+) deletes (\S+) from artifact store + user = get_next_match() + path = get_next_match() + token = get_token(user) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, delete, asurl + path, token=token) IMPLEMENTS WHEN (\S+) makes request GET (\S+) with an invalid token user = get_next_match() @@ -54,6 +66,8 @@ along with this program. If not, see . user = get_next_match() path = get_next_match() body = get_next_match() + body = expand_vars(body, V) + V['xxxPOSTbodyvalid'] = body token = get_token(user) url = V['url'] http(V, post, url + path, body=body, token=token) @@ -62,6 +76,8 @@ along with this program. If not, see . user = get_next_match() path = get_next_match() body = get_next_match() + body = expand_vars(body, V) + V['xxxPOSTbody'] = body token = get_token(user) url = V['url'] http(V, post, url + path, body=body, token='invalid') @@ -69,7 +85,10 @@ along with this program. If not, see . IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with a valid token and body (.+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) body = get_next_match() + body = expand_vars(body, V) + V['xxxPUTbody'] = body token = get_token(user) url = V['url'] http(V, put, url + path, body=body, token=token) @@ -80,12 +99,15 @@ along with this program. If not, see . path = get_next_match() body = cat(filename) token = get_token(user) - url = V['bsurl'] - http(V, put_blob, url + path, body=body, token=token) + url = V['url'] + version = get_version(url) + asurl = version['artifact_store'] + http(V, put_blob, asurl + path, body=body, token=token) IMPLEMENTS WHEN (\S+) makes request PUT (\S+) with an invalid token user = get_next_match() path = get_next_match() + path = expand_vars(path, V) body = '{}' token = get_token(user) url = V['url'] @@ -94,18 +116,25 @@ along with this program. If not, see . IMPLEMENTS WHEN (\S+) makes request DELETE (\S+) user = get_next_match() path = get_next_match() + path = expand_vars(path, V) token = get_token(user) url = V['url'] http(V, delete, url + path, token=token) ## HTTP response inspection + IMPLEMENTS THEN worker id is (\S+) + varname = get_next_match() + body = json.loads(V['body']) + V[varname] = body['worker'] + IMPLEMENTS THEN result has status code (\d+) expected = int(get_next_match()) assertEqual(expected, V['status_code']) IMPLEMENTS THEN body matches (.+) expected_text = get_next_match() + expected_text = expand_vars(expected_text, V) expected = json.loads(expected_text) actual = json.loads(V['body']) print 'expected' diff --git a/yarns/900-local.yarn b/yarns/900-local.yarn deleted file mode 100644 index e9e8a7c..0000000 --- a/yarns/900-local.yarn +++ /dev/null @@ -1,121 +0,0 @@ - - -# Scenario step implementations for locally managed ick - -## Authentication setup - - IMPLEMENTS GIVEN an RSA key pair for token signing - argv = [ - os.path.join(srcdir, 'generate-rsa-key'), - 'token.key', - ] - cliapp.runcmd(argv, stdout=None, stderr=None) - - IMPLEMENTS GIVEN an access token for (\S+) with scopes (.+) - user = get_next_match() - scopes = get_next_match() - key = open('token.key').read() - argv = [ - os.path.join(srcdir, 'create-token'), - scopes, - user, - ] - token = cliapp.runcmd(argv, feed_stdin=key) - store_token(user, token) - V['issuer'] = 'localhost' - V['audience'] = user - -## Controller configuration - - IMPLEMENTS GIVEN controller config uses (\S+) at the state directory - V['statedir'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as artifact store - V['artifact_store'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as authentication - V['auth_url'] = get_next_match() - - IMPLEMENTS GIVEN controller config uses (\S+) as notify - V['notify_url'] = get_next_match() - assert V['notify_url'] is not None - -## Start and stop the controller - - IMPLEMENTS GIVEN a running ick controller - start_controller() - - IMPLEMENTS WHEN user stops ick controller - stop_controller() - - IMPLEMENTS FINALLY stop ick controller - stop_controller() - -## Controller state inspection - - IMPLEMENTS THEN controller state directory contains project (\S+) - name = get_next_match() - basename = encode_basename(name) - filename = os.path.join(V['statedir'], 'projects', basename) - print 'name', name - print 'basename', basename - print 'filename', filename - assertTrue(os.path.exists(filename)) - - IMPLEMENTS THEN controller state directory contains worker (\S+) - name = get_next_match() - basename = encode_basename(name) - filename = os.path.join(V['statedir'], 'workers', basename) - print 'filename', filename - assertTrue(os.path.exists(filename)) - -## Check version result - - IMPLEMENTS THEN artifact store URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['artifact_store'] - assertEqual(actual, expected) - - IMPLEMENTS THEN authentication URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['auth_url'] - assertEqual(actual, expected) - - IMPLEMENTS THEN notify URL is (\S+) - expected = get_next_match() - body = V['body'] - obj = json.loads(body) - actual = obj['notify_url'] - assertEqual(actual, expected) - -## Start and stop artifact store - - IMPLEMENTS GIVEN artifact store config uses (\S+) at the blob directory - V['blobdir'] = get_next_match() - - IMPLEMENTS GIVEN a running artifact store - start_artifact_store() - - IMPLEMENTS FINALLY stop artifact store - stop_artifact_store() diff --git a/yarns/900-remote.yarn b/yarns/900-remote.yarn index 3c0443c..5e84c13 100644 --- a/yarns/900-remote.yarn +++ b/yarns/900-remote.yarn @@ -21,22 +21,12 @@ along with this program. If not, see . ## Authentication setup - IMPLEMENTS GIVEN an RSA key pair for token signing - V['private_key_file'] = os.environ['ICK_PRIVATE_KEY'] - assertTrue(os.path.exists(V['private_key_file'])) - IMPLEMENTS GIVEN an access token for (\S+) with scopes (.+) user = get_next_match() - scopes = get_next_match() - key = open(V['private_key_file']).read() - argv = [ - os.path.join(srcdir, 'create-token'), - scopes, - ] - token = cliapp.runcmd(argv, feed_stdin=key) + scopes = get_next_match().split() + create_api_client(user, scopes) + token = get_api_token(user, scopes) store_token(user, token) - V['issuer'] = 'localhost' - V['audience'] = 'localhost' ## Controller configuration @@ -46,13 +36,14 @@ along with this program. If not, see . ## Start and stop the controller IMPLEMENTS GIVEN a running ick controller - V['url'] = os.environ['ICK_URL'] + V['url'] = os.environ['CONTROLLER'] IMPLEMENTS WHEN user stops ick controller pass IMPLEMENTS FINALLY stop ick controller - pass + for client_id in get_client_ids(): + delete_api_client(client_id) ## Controller state inspection diff --git a/yarns/lib.py b/yarns/lib.py index 5fbd5ab..6d8f2cf 100644 --- a/yarns/lib.py +++ b/yarns/lib.py @@ -19,11 +19,13 @@ import errno import json import os import random +import re import signal import socket import sys import time import urllib +import uuid import cliapp import requests @@ -37,119 +39,80 @@ datadir = os.environ['DATADIR'] V = Variables(datadir) -def start_controller(): - port = V['port'] = random_free_port() - - V['url'] = 'http://127.0.0.1:{}'.format(V['port']) - - filename = 'ick_controller.yaml' - env = dict(os.environ) - env['ICK_CONTROLLER_CONFIG'] = filename - write_yaml(filename, { - 'token-issuer': V['issuer'], - 'token-audience': V['audience'], - 'token-public-key': cat('token.key.pub'), - 'log': [ - { - 'filename': 'ick_controller.log', - }, - ], - 'statedir': V['statedir'], - 'apt-server': 'localhost', - 'artifact-store': V['artifact_store'], - 'auth-url': V['auth_url'], - 'notify-url': V['notify_url'], - }) - - V['pid'] = gunicorn('ick_controller', 'app', port, env) - - -def stop_controller(): - if V['pid'] is not None: - os.kill(int(V['pid']), signal.SIGTERM) - - -def start_artifact_store(): - port = V['bsport'] = random_free_port() - - V['bsurl'] = 'http://127.0.0.1:{}'.format(V['bsport']) - - filename = 'artifact_store.yaml' - env = dict(os.environ) - env['ARTIFACT_STORE_CONFIG'] = filename - write_yaml(filename, { - 'token-issuer': V['issuer'], - 'token-audience': V['audience'], - 'token-public-key': cat('token.key.pub'), - 'log': [ - { - 'filename': 'artifact_store.log', - }, - ], - 'blobdir': V['blobdir'], - }) - - V['bspid'] = gunicorn('artifact_store', 'app', port, env) - - -def stop_artifact_store(): - if V['pid'] is not None: - os.kill(int(V['bspid']), signal.SIGTERM) - - -def write_yaml(filename, obj): - yaml.safe_dump(obj, open(filename, 'w')) - - -def gunicorn(module_name, var_name, port, env): - log_filename = '{}.gunicorn.log'.format(module_name) - pid_filename = '{}.pid'.format(module_name) - - argv = [ - 'gunicorn3', - '--daemon', - '--bind', '127.0.0.1:{}'.format(port), - '--log-file', log_filename, - '--log-level', 'debug', - '-p', pid_filename, - '{}:{}'.format(module_name, var_name), - ] - cliapp.runcmd(argv, env=env) - wait_for_port(port) - return int(cat(pid_filename)) - - -def random_free_port(): - MAX = 1000 - for i in range(MAX): - port = random.randint(1025, 2**15-1) - s = socket.socket() - try: - s.bind(('0.0.0.0', port)) - except OSError as e: - if e.errno == errno.EADDRINUSE: - continue - print('cannot find a random free port') - raise - s.close() - break - print('picked port', port) - return port - - -def wait_for_port(port): - MAX = 5 - t = time.time() - while time.time() < t + MAX: - try: - s = socket.socket() - s.connect(('127.0.0.1', port)) - except socket.error: - time.sleep(0.1) - except OSError as e: - raise - else: - return +def remember_client_id(alias, client_id, client_secret): + clients = V['clients'] + if clients is None: + clients = {} + clients[alias] = { + 'client_id': client_id, + 'client_secret': client_secret, + } + V['clients'] = clients + + +def get_client_id(alias): + clients = V['clients'] or {} + return clients[alias]['client_id'] + + +def get_client_ids(): + clients = V['clients'] or {} + return [x['client_id'] for x in clients.values()] + + +def get_client_secret(alias): + clients = V['clients'] or {} + return clients[alias]['client_secret'] + + +def create_api_client(alias, scopes): + client_id = str(uuid.uuid4()) + client_secret = str(uuid.uuid4()) + print('invented client id', client_id) + api = os.environ['CONTROLLER'] + print('controller URL', api) + secrets = os.environ['SECRETS'] + print('secrets', secrets) + base_argv = ['qvisqvetool', '--secrets', secrets, '-a', api] + print('base_argv', base_argv) + cliapp.runcmd(base_argv + ['create', 'client', client_id, client_secret]) + cliapp.runcmd(base_argv + ['allow-scope', 'client', client_id] + scopes) + remember_client_id(alias, client_id, client_secret) + + +def delete_api_client(client_id): + api = os.environ['CONTROLLER'] + secrets = os.environ['SECRETS'] + base_argv = ['qvisqvetool', '--secrets', secrets, '-a', api] + cliapp.runcmd(base_argv + ['delete', 'client', client_id]) + + +def get_api_token(alias, scopes): + print('getting token for', alias) + + client_id = get_client_id(alias) + client_secret = get_client_secret(alias) + api = os.environ['CONTROLLER'] + + auth = (client_id, client_secret) + data = { + 'grant_type': 'client_credentials', + 'scope': ' '.join(scopes), + } + + url = '{}/token'.format(api) + + print('url', url) + print('auth', auth) + print('data', data) + r = requests.post(url, auth=auth, data=data) + if not r.ok: + sys.exit('Error getting token: %s %s' % (r.status_code, r.text)) + + token = r.json()['access_token'] + print('token', token) + return token + def unescape(s): t = '' @@ -186,6 +149,12 @@ def get_token(user): def http(V, func, url, **kwargs): + V['request'] = { + 'func': repr(func), + 'url': url, + 'kwargs': kwargs, + } + print('http', func, url, kwargs) status, content_type, headers, body = func(url, **kwargs) V['status_code'] = status V['content_type'] = content_type @@ -201,6 +170,11 @@ def get(url, token): return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text +def get_version(url): + status, ctype, headers, text = get(url + '/version', 'no token') + assert ctype == 'application/json' + return json.loads(text) + def get_blob(url, token): headers = { 'Authorization': 'Bearer {}'.format(token), @@ -309,5 +283,15 @@ def list_diff(a, b): return None -def encode_basename(basename): - return urllib.quote(basename, safe='') +def expand_vars(text, variables): + result = '' + while text: + m = re.search(r'\${(?P[^}]+)}', text) + if not m: + result += text + break + name = m.group('name') + print('expanding ', name) + result += text[:m.start()] + variables[name] + text = text[m.end():] + return result -- cgit v1.2.1