diff options
Diffstat (limited to 'vcsworker.py')
| -rwxr-xr-x | vcsworker.py | 312 |
1 files changed, 0 insertions, 312 deletions
diff --git a/vcsworker.py b/vcsworker.py deleted file mode 100755 index 5983268..0000000 --- a/vcsworker.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python3 -# -# This implements the various APIs for the various CI components we -# need. Very simplistic and prototype-y. All in one file for -# simplicity - production version will need to be done more carefully. -# -# To use: run this program with one of the following command lines: -# -# ./this controller this.log key.pub -# ./this vcsworker this.log key.pub gitlab.token -# -# where -# -# - this.log is the log file to use -# - key.pub is a public key in ssh format for checking incoming access tokens -# - gitlab.token is the name of a file containing a GitLab access token -# -# NOTE: The GitLab instance and other configuration details are -# hardcoded in this version. This will be fixed, as well as the 418 -# HTTP status code as a generic error code. -# -# NOTE: You should run this behind haproxy or smilar TLS provider, -# which forwards requests to localhost. - - -import logging -import os -import shutil -import subprocess -import sys -import tempfile -import urllib.parse - -import Crypto.PublicKey.RSA -import bottle -import jwt - - -HOST = 'localhost' -PORT = 2222 - - -class TokenParser: - - '''Parse an incoming access token (signed JWT)''' - - # Note that if we need to, for performance, we can cache the parse - # results here. But there's no point in doing that unless it - # becomes necessary. - - def __init__(self, pubkey): - self._pubkey = pubkey - - def parse_token(self, token_text): - return jwt.decode( - token_text, - key=self._pubkey.exportKey('OpenSSH'), - audience=None, - options={'verify_aud': False}) - - -class AccessChecker: - - '''Given request headers and required scopes, is a request allowed?''' - - def __init__(self, pubkey): - self._parser = TokenParser(pubkey) - - def access_is_allowed(self, headers, required_scopes): - token = self._get_token(headers) - logging.debug('Access token %r', token) - - if token is None and len(required_scopes) != 0: - logging.error('No valid access token') - return False - - if token: - scopes = token.get('scope', '').split() - missing = set(required_scopes).difference(scopes) - if missing: - logging.error( - 'Required scopes that are missing from token: %r', missing) - return False - - return True - - def _get_token(self, headers): - '''Parse an access token or return None if it's bad''' - token_text = self._get_token_text(headers) - if token_text is None: - return None - return self._parser.parse_token(token_text) - - def _get_token_text(self, headers): - '''Extract access token from request headers or None if not there''' - v = headers.get('Authorization', '') - words = v.split() - if len(words) == 2: - keyword, token_text = words - if keyword.lower() == 'bearer': - return token_text - - -class API: - - '''Base class for simple HTTP APIs - - Override the get_routes method to use this. Call setup() method to - set up routes and such, before actually running. - - ''' - - def __init__(self): - self._checker = None - - def setup(self, app, token_pubkey): - self._checker = AccessChecker(token_pubkey) - self._add_routes(app, self.get_routes()) - - def get_routes(self): - raise NotImplementedError() - - def _add_routes(self, app, routes): - for route in routes: - func = route.pop('func') - scopes = route.pop('scopes') - assert isinstance(scopes, list) - callback = lambda **kwargs: self.check(func, scopes, kwargs) - route = dict(route) - route['callback'] = callback - app.route(**route) - - def check(self, func, required_scopes, kwargs): - '''Call a callback function, if it's OK to do so''' - r = bottle.request - logging.debug('New request, checking access: %s %s', r.method, r.path) - - if self._checker.access_is_allowed(r.headers, required_scopes): - logging.info('Access is allowed: %s %s', r.method, r.path) - ret = func(**kwargs) - logging.info('Result: %r', ret) - return ret - - logging.error('Request denied %s %s', r.method, r.path) - return bottle.HTTPError(400) - - -class Controller(API): - - '''A dummy controller API''' - - def get_routes(self): - return [ - { - 'method': 'GET', - 'path': '/status', - 'func': self._status, - 'scopes': ['status'], - }, - { - 'method': 'GET', - 'path': '/hello/<name>', - 'func': self._hello, - 'scopes': ['hello'], - }, - ] - - def _status(self): - return { - 'queue': [], - 'running': [], - 'finished': [], - } - - def _hello(self, name=None): - return 'hello {}\n'.format(name) - - -class VCSWorker(API): - - '''A VCSWorker API''' - - MAX_CLONE_TIME = 1 - MAX_REMOVE_TIME = 60 - MAX_PUSH_TIME = 60 - GITLAB_DOMAIN = 'wmf-gitlab3.vm.liw.fi' - GITLAB_PROJECT = 'liw' - - def __init__(self, gitlab_token): - self._token = gitlab_token - self._tmpdir = tempfile.mkdtemp() - logging.info('Workspace: %s', self._tmpdir) - - def get_routes(self): - return [ - { - 'method': 'POST', - 'path': '/updaterepo', - 'func': self._update_repo, - 'scopes': ['update-repo'], - }, - ] - - def _update_repo(self): - spec = bottle.request.json - logging.info('Updating repository: %r', spec) - - url = spec['git'] - ref = spec['ref'] - name = spec['gitlab'] - dirname = os.path.join(self._tmpdir, name) - - if self._clone(url, ref, dirname): - ok = self._remove( - self._token, self.GITLAB_DOMAIN, self.GITLAB_PROJECT, name) - if ok: - ok = self._push( - dirname, self.GITLAB_DOMAIN, self.GITLAB_PROJECT, ref, - name) - if ok: - return 'Repository copied successfully' - - logging.error('Something went wrong when copying repository') - return bottle.HTTPError(418) - - def _clone(self, url, ref, dirname): - if os.path.exists(dirname): - logging.debug('Removing %s', dirname) - shutil.rmtree(dirname) - argv = ['git', 'clone', '-q', '-b', ref, url, dirname] - return runcmd('.', argv, self.MAX_CLONE_TIME) - - def _remove(self, token, gitlab_domain, gitlab_project, name): - snippet = urllib.parse.quote('%s/%s' % (gitlab_project, name), safe='') - url = 'https://%s/api/v4/projects/%s' % (gitlab_domain, snippet) - argv = ['curl', '-HPRIVATE-TOKEN: %s' % token, '-X', 'DELETE', url] - return runcmd('.', argv, self.MAX_REMOVE_TIME) - - def _push(self, dirname, gitlab_domain, gitlab_project, ref, name): - logging.info('Pushing %s to %s as %s', dirname, gitlab_domain, name) - url = 'ssh://git@%s/%s/%s.git' % (gitlab_domain, gitlab_project, name) - argv = ['git', 'push', url, '%s:master' % ref] - return runcmd(dirname, argv, self.MAX_PUSH_TIME) - - -def runcmd(cwd, argv, timeout): - logging.info('Running command: %r', argv) - try: - p = subprocess.run( - argv, cwd=cwd, timeout=timeout, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except subprocess.TimeoutExpired: - logging.error('Command took too long (timeout %r)', timeout) - return False - except Exception as e: - logging.error('Error while running command: %s', str(e)) - return False - - if p.returncode != 0: - logging.error('Command failed: %r', argv) - logging.error('exit code: %d', p.returncode) - logging.error('stdout: %r', p.stdout) - logging.error('stderr: %r', p.stderr) - return False - - logging.info('Command succeeded') - return True - - -def setup_logging(log_filename): - logging.basicConfig( - filename=log_filename, - level=logging.DEBUG, - format='%(asctime)s %(levelname)s %(message)s') - logging.info('API server starting') - - -def get_key_from_file(filename): - with open(filename) as f: - key_text = f.read() - return Crypto.PublicKey.RSA.importKey(key_text) - - -def get_token_from_file(filename): - with open(filename) as f: - return f.read().strip() - - -def main(): - args = sys.argv[1:] - cmd = args.pop(0) - log_filename = args.pop(0) - pubkey_filename = args.pop(0) - - setup_logging(log_filename) - - if cmd == 'controller': - api = Controller() - elif cmd == 'vcsworker': - gitlab_token_filename = args.pop(0) - gitlab_token = get_token_from_file(gitlab_token_filename) - api = VCSWorker(gitlab_token) - else: - sys.exit('Unknown command %s' % cmd) - - app = bottle.Bottle() - pubkey = get_key_from_file(pubkey_filename) - api.setup(app, pubkey) - app.run(host=HOST, port=PORT) - - -main() |