From cac861f9a8a61912bfe92fbf51fedf6e65d737e9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 1 Jan 2019 15:03:12 +0200 Subject: Add: scaffolding for yarns --- check | 7 +++ effiapi | 163 +++++++++++++++++++++++++++++++++++++++++++++----------- yarns/000.yarn | 87 +++++++++++++++++++++++++----- yarns/lib.py | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 379 insertions(+), 43 deletions(-) create mode 100755 check create mode 100644 yarns/lib.py diff --git a/check b/check new file mode 100755 index 0000000..420f433 --- /dev/null +++ b/check @@ -0,0 +1,7 @@ +#!/bin/sh + +set -eu + +yarn yarns/*.yarn \ + --shell python2 --shell-arg '' --shell-library yarns/lib.py \ + --cd-datadir "$@" diff --git a/effiapi b/effiapi index cb84fe3..ab3640f 100755 --- a/effiapi +++ b/effiapi @@ -25,6 +25,87 @@ import bottle import requests +class HTTPAPI: + + def POST(self, url, headers, body): + raise NotImplementedError() + + def PUT(self, url, headers, body): + raise NotImplementedError() + + def GET(self, url, headers=None, body=None): + raise NotImplementedError() + + def DELETE(self, url, headers): + raise NotImplementedError() + + +class RealHTTPAPI(HTTPAPI): + + def POST(self, url, headers, body): + assert 0 + return requests.post(url, headers=headers, data=body) + + def PUT(self, url, headers, body): + return requests.put(url, headers=headers, data=body) + + def GET(self, url, headers=None, body=None): + return requests.get(url, headers=headers, data=body) + + def DELETE(self, url, headers): + return requests.delete(url, headers=headers, data=body) + + +class FakeResponse: + + def __init__(self, status_code, headers, body): + self.status_code = status_code + self.headers = dict(headers) + self.text = body + + @property + def ok(self): + return self.status_code in (200, 201) + + def json(self): + return json.loads(self.text.strip()) + + +class FakeHTTPAPI(HTTPAPI): + + def __init__(self): + self._memb = {} + + def POST(self, url, headers, body): + rid = str(uuid.uuid4()) + self._memb[rid] = copy.deepcopy(body) + headers = { + 'Muck-Id': rid, + } + return FakeResponse(201, headers, copy.deepcopy(body)) + + def PUT(self, url, headers, body): + raise NotImplementedError() + + def GET(self, url, headers=None, body=None): + if headers is None: + return FakeRespone(400, {}, 'Missing headers') + logging.debug('GET headers %r', headers) + rid = headers.get('Muck-Id') + logging.info('GET for %r', rid) + if not rid: + return FakeResponse(404, {}, 'No such member') + memb = self._memb[rid] + headers = { + 'Muck-Id': rid, + 'Content-Type': 'application/json', + } + return FakeResponse(200, headers, memb) + + def DELETE(self, url, headers): + raise NotImplementedError() + + class MuckError(Exception): pass @@ -39,8 +120,9 @@ class MuckAPI: 'Muck-Owner', ] - def __init__(self, url): + def __init__(self, url, httpapi): self._url = url + self._httpapi = httpapi def _get_headers(self): h = {} @@ -56,13 +138,15 @@ class MuckAPI: def status(self): url = self.url('/status') - r = requests.get(url) + r = self._httpapi.GET(url) return r.json() def show(self, rid): url = self.url('/res') headers = self._get_headers() - r = requests.get(url, headers=headers) + headers['Muck-Id'] = rid + logging.info('show copied headers') + r = self._httpapi.GET(url, headers=headers) logging.info('Show result: %s %s', r.status_code, r.text) if not r.ok: raise MuckError('{} {}'.format(r.status_code, r.text)) @@ -72,17 +156,20 @@ class MuckAPI: url = self.url('/res') headers = self._get_headers() headers['Content-Type'] = 'application/json' - r = requests.post(url, headers=headers, data=json.dumps(member)) + r = self._httpapi.POST(url, headers, json.dumps(member)) logging.info('Show result: %s %s', r.status_code, r.text) if not r.ok: raise MuckError('{} {}'.format(r.status_code, r.text)) - return r.json() + rid = r.headers.get('Muck-Id') + if not rid: + raise MuckError('Muck did not return Muck-Id') + return rid, r.json() def search(self, cond): url = self.url('/search') headers = self._get_headers() headers['Content-Type'] = 'application/json' - r = requests.get(url, data=json.dumps(cond), headers=headers) + r = self._httpapi.GET(url, headers=headers, body=json.dumps(cond)) logging.info('Search result: %s %s', r.status_code, r.text) if not r.ok: raise MuckError('{} {}'.format(r.status_code, r.text)) @@ -91,9 +178,9 @@ class MuckAPI: class API: - def __init__(self, bottleapp, config): + def __init__(self, httpapi, bottleapp, config): self._add_routes(bottleapp) - self._muck = MuckAPI(config['muck-url']) + self._muck = MuckAPI(config['muck-url'], httpapi) def _add_routes(self, bottleapp): routes = [ @@ -104,7 +191,7 @@ class API: }, { 'method': 'GET', - 'path': '/mem', + 'path': '/memb', 'callback': self._call(self._show_member), }, { @@ -114,7 +201,7 @@ class API: }, { 'method': 'POST', - 'path': '/mem', + 'path': '/memb', 'callback': self._call(self._create), }, ] @@ -124,13 +211,20 @@ class API: def _call(self, callback): def helper(): - r = bottle.request - logging.info('Request: path=%s', r.path) - logging.info('Request: content type: %s', r.content_type) - for h in r.headers: - logging.info('Request: headers: %s: %s', h, r.get_header(h)) - logging.info('Request: body: %r', r.body.read()) - return callback() + try: + r = bottle.request + logging.info('Request: method=%s', r.method) + logging.info('Request: path=%s', r.path) + logging.info('Request: content type: %s', r.content_type) + for h in r.headers: + logging.info('Request: headers: %s: %s', h, r.get_header(h)) + logging.info('Request: body: %r', r.body.read()) + ret = callback() + except BaseException as e: + logging.error(str(e)) + raise + else: + return ret return helper def _get_token(self): @@ -147,38 +241,41 @@ class API: def _show_status(self): status = self._muck.status() - return response(200, status) + return response(200, None, status) def _show_member(self): rid = bottle.request.get_header('Muck-Id') if rid is None: - return response(400) + return response(400, None, 'no muck-id') try: + logging.debug('API _show_member rid=%r', rid) res = self._muck.show(rid) - return response(200, res) + return response(200, rid, res) except MuckError as e: - return response(404, str(e)) + return response(404, None, str(e)) def _search(self): cond = bottle.request.json result = self._muck.search(cond) - return response(200, result) + return response(200, None, result) def _create(self): r = bottle.request if r.content_type != 'application/json': - return response(400) + return response(400, None, 'wrong content type') obj = bottle.request.json logging.info('CREATE %r', repr(obj)) - newobj = self._muck.create(obj) - return response(201, newobj) + rid, newobj = self._muck.create(obj) + return response(201, rid, newobj) -def response(status, body): - headers = {} - if isinstance(body, dict): - headers['Content-Type'] = 'application/json' +def response(status, rid, body): + headers = { + 'Content-Type': 'application/json', + } + if rid is not None: + headers['Muck-Id'] = rid return bottle.HTTPResponse( status=status, body=json.dumps(body), headers=headers) @@ -197,6 +294,12 @@ if config.get('pid'): with open(config['pid'], 'w') as f: f.write(str(pid)) +if config.get('fake', False): + httpapi = RealHTTPAPI() + assert 0 +else: + httpapi = FakeHTTPAPI() + app = bottle.default_app() -api = API(app, config) +api = API(httpapi, app, config) app.run(host='127.0.0.1', port=8080) diff --git a/yarns/000.yarn b/yarns/000.yarn index 9bd0432..1c3f55e 100644 --- a/yarns/000.yarn +++ b/yarns/000.yarn @@ -3,11 +3,14 @@ title: Effiapi test suite author: Lars Wirzenius ... -# Test scenarios +[yarn]: https://liw.fi/cmdtest/ + +# Introduction -This chapter descibes the effiapi API, as a [yarn][] automated -scenario test. It is meant to be understandable to Effi sysadmins, as -well as be an executable test suite for the API. +This chapter descibes the effiapi API, using [yarn][] automated +scenario tests. It is meant to be understandable to Effi sysadmins, +and those writing applications using the API, as well as be an +executable test suite for the API. The API is a simple RESTful HTTP API using JSON. This means that: @@ -24,22 +27,27 @@ The API is a simple RESTful HTTP API using JSON. This means that: * standard HTTP status codes are used to indicate result of the request (200 = OK, 404 = not found, etc) +Examples will be provided. + +# Test scenarios + ## Manage memberships This section shows the API calls to manage a memberhip: to create the member, to update and retrieve it, and to search memberships. -~~~ -SCENARIO Manage memberships + SCENARIO Manage memberships + + GIVEN An effiapi instance + WHEN admin requests POST /memb with body { "fullname": "James Bond" } + THEN HTTP status is 201 + AND the member id is ID -GIVEN An effiapi instance -WHEN admin requests POST /memb with body { "fullname": "James Bond" } -THEN the member id is ID + WHEN admin requests GET /memb with header Muck-Id: ${ID} + THEN HTTP status is 200 + AND HTTP body matches { "fullname": "James Bond" } -WHEN admin requests GET /memb with header Muck-Id: ${ID} -THEN HTTP status 200 -AND HTTP body matches { "fullname": "James Bond" } -~~~ + FINALLY Effiapi is terminated TODO: @@ -50,3 +58,56 @@ TODO: * member follows authn link emailed to them # Appendix: Yarn scenario step implementations + +## Start and stop effiapi + + IMPLEMENTS GIVEN An effiapi instance + effiapi.write_config() + effiapi.start() + + IMPLEMENTS FINALLY Effiapi is terminated + effiapi.terminate() + +## Make HTTP requests + + IMPLEMENTS WHEN admin requests POST /memb with body (.+) + body = get_json_match() + effiapi.POST('/memb', {}, body) + + IMPLEMENTS WHEN admin requests GET /memb with header (\S+): (\S+) + header = get_next_match() + print('header', repr(header)) + value = get_expanded_match() + print('value', repr(value)) + headers = { + header: value, + } + V['xx'] = { + 'header': header, + 'value': value, + } + effiapi.GET('/memb', headers, None) + +## Inspect HTTP responses + + IMPLEMENTS THEN the member id is (\S+) + print('member id') + name = get_next_match() + print 'name', repr(name), name + value = effiapi.get_header('Muck-Id') + print 'value', repr(value) + save_for_expansion(name, value) + + IMPLEMENTS THEN HTTP status is (\d+) + expected = int(get_next_match()) + actual = effiapi.get_status_code() + print 'actual:', repr(actual) + print 'expecting:', repr(expected) + assertEqual(effiapi.get_status_code(), expected) + + IMPLEMENTS THEN HTTP body matches (.+) + expected = get_json_match() + actual = effiapi.get_json_body() + print 'expected:', expected + print 'actual: ', actual + assertEqual(actual, expected) diff --git a/yarns/lib.py b/yarns/lib.py new file mode 100644 index 0000000..d7c8a13 --- /dev/null +++ b/yarns/lib.py @@ -0,0 +1,165 @@ +# Copyright 2019 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import base64 +import errno +import json +import os +import random +import re +import signal +import socket +import subprocess +import sys +import time +import urllib + +import cliapp +import requests + +from yarnutils import * + + + +class EffiAPI: + + config_filename = 'effiapi.json' + + def __init__(self, variables): + self.v = variables + + def write_config(self): + with open(self.config_filename, 'w') as f: + json.dump(self.get_config(), f, indent=4) + + def get_config(self): + return { + 'faks': True, + 'muck-url': 'xxx', + 'log': 'effiapi.log', + 'pid': 'effiapi.pid', + 'host': '127.0.0.1', + 'port': 8080, + } + + def start(self): + config = self.get_config() + self.v['baseurl'] = 'http://{}:{}'.format( + config['host'], config['port']) + + effiapi = os.path.join(srcdir, 'effiapi') + self.daemonize([effiapi, self.config_filename]) + + self.wait_for_port(config['host'], config['port']) + + def terminate(self): + config = self.get_config() + with open(config['pid']) as f: + pid = int(f.read().strip()) + os.kill(pid, signal.SIGTERM) + + def daemonize(self, argv): + prefix = [ + '/usr/sbin/daemonize', '-o', 'stdout', '-e', 'stderr', '-c', '.' + ] + subprocess.check_call(prefix + argv) + + def wait_for_port(self, host, port): + MAX = 5 + t = time.time() + while time.time() < t + MAX: + try: + s = socket.socket() + s.connect((host, port)) + except socket.error: + time.sleep(0.1) + except OSError as e: + raise + else: + return + + def POST(self, path, headers, body): + headers['Content-Type'] = 'application/json' + body = json.dumps(body) + self.request(requests.post, path, headers, body) + + def GET(self, path, headers, body): + self.request(requests.get, path, headers, body) + + def request(self, func, path, headers, body): + url = '{}{}'.format(self.v['baseurl'], path) + self.v['request'] = { + 'url': url, + 'func': repr(func), + 'headers': headers, + 'body': body, + } + + r = func(url, headers=headers, data=body) + self.v['response'] = { + 'status_code': r.status_code, + 'body': r.text, + 'headers': dict(r.headers), + } + + def get_status_code(self): + r = self.v['response'] + return r['status_code'] + + def get_header(self, name): + r = self.v['response'] + return r['headers'].get(name, '') + + def get_json_body(self): + r = self.v['response'] + return json.loads(r['body']) + + +def get_json_match(): + match = get_next_match() + return json.loads(match) + + +def save_for_expansion(name, value): + V[name] = value + + +def get_expanded_match(): + match = get_next_match() + print 'match', match + return expand(match, V) + + +def expand(text, variables): + result = '' + while text: + print 'expand: text=%r' % text + m = re.search(r'\${(?P[^}]+)}', text) + if not m: + result += text + break + name = m.group('name') + print('expanding ', name, repr(variables[name])) + result += text[:m.start()] + variables[name] + text = text[m.end():] + print 'expand: result=%r' % result + return result + + +srcdir = os.environ['SRCDIR'] +datadir = os.environ['DATADIR'] +V = Variables(datadir) +effiapi = EffiAPI(V) -- cgit v1.2.1