Rename: vcsworker.py to api.py

1 files changed, 186 insertions, 13 deletions

diff --git a/api.py b/api.py
index d593260..5983268 100644..100755
--- a/api.py
+++ b/api.py
@@ -1,13 +1,53 @@
+#!/usr/bin/env python3
+#
+# This implements the various APIs for the various CI components we
+# need. Very simplistic and prototype-y. All in one file for
+# simplicity - production version will need to be done more carefully.
+#
+# To use: run this program with one of the following command lines:
+#
+#       ./this controller this.log key.pub 
+#       ./this vcsworker this.log key.pub gitlab.token
+#
+# where
+#
+# - this.log is the log file to use
+# - key.pub is a public key in ssh format for checking incoming access tokens
+# - gitlab.token is the name of a file containing a GitLab access token
+#
+# NOTE: The GitLab instance and other configuration details are
+# hardcoded in this version. This will be fixed, as well as the 418
+# HTTP status code as a generic error code.
+#
+# NOTE: You should run this behind haproxy or smilar TLS provider,
+# which forwards requests to localhost.
+
+
 import logging
+import os
+import shutil
+import subprocess
 import sys
+import tempfile
+import urllib.parse
 
 import Crypto.PublicKey.RSA
 import bottle
 import jwt
 
 
+HOST = 'localhost'
+PORT = 2222
+
+
 class TokenParser:
 
+    '''Parse an incoming access token (signed JWT)'''
+
+    # Note that if we need to, for performance, we can cache the parse
+    # results here. But there's no point in doing that unless it
+    # becomes necessary.
+
     def __init__(self, pubkey):
         self._pubkey = pubkey
 
@@ -21,12 +61,15 @@ class TokenParser:
 
 class AccessChecker:
 
+    '''Given request headers and required scopes, is a request allowed?'''
+
     def __init__(self, pubkey):
         self._parser = TokenParser(pubkey)
 
     def access_is_allowed(self, headers, required_scopes):
         token = self._get_token(headers)
         logging.debug('Access token %r', token)
+
         if token is None and len(required_scopes) != 0:
             logging.error('No valid access token')
             return False
@@ -42,12 +85,14 @@ class AccessChecker:
         return True
 
     def _get_token(self, headers):
+        '''Parse an access token or return None if it's bad'''
         token_text = self._get_token_text(headers)
         if token_text is None:
             return None
         return self._parser.parse_token(token_text)
 
     def _get_token_text(self, headers):
+        '''Extract access token from request headers or None if not there'''
         v = headers.get('Authorization', '')
         words = v.split()
         if len(words) == 2:
@@ -58,7 +103,17 @@ class AccessChecker:
 
 class API:
 
-    def __init__(self, app, token_pubkey):
+    '''Base class for simple HTTP APIs
+
+    Override the get_routes method to use this. Call setup() method to
+    set up routes and such, before actually running.
+
+    '''
+
+    def __init__(self):
+        self._checker = None
+
+    def setup(self, app, token_pubkey):
         self._checker = AccessChecker(token_pubkey)
         self._add_routes(app, self.get_routes())
 
@@ -69,17 +124,19 @@ class API:
         for route in routes:
             func = route.pop('func')
             scopes = route.pop('scopes')
+            assert isinstance(scopes, list)
             callback = lambda **kwargs: self.check(func, scopes, kwargs)
             route = dict(route)
             route['callback'] = callback
             app.route(**route)
 
     def check(self, func, required_scopes, kwargs):
+        '''Call a callback function, if it's OK to do so'''
         r = bottle.request
-        logging.debug('Checking access for request %s %s', r.method, r.path)
+        logging.debug('New request, checking access: %s %s', r.method, r.path)
 
-        if self._checker.access_is_allowed(bottle.request.headers, required_scopes):
-            logging.info('Serving request %s %s', r.method, r.path)
+        if self._checker.access_is_allowed(r.headers, required_scopes):
+            logging.info('Access is allowed: %s %s', r.method, r.path)
             ret = func(**kwargs)
             logging.info('Result: %r', ret)
             return ret
@@ -90,6 +147,8 @@ class API:
 
 class Controller(API):
 
+    '''A dummy controller API'''
+
     def get_routes(self):
         return [
             {
@@ -117,23 +176,137 @@ class Controller(API):
         return 'hello {}\n'.format(name)
 
 
+class VCSWorker(API):
+
+    '''A VCSWorker API'''
+
+    MAX_CLONE_TIME = 1
+    MAX_REMOVE_TIME = 60
+    MAX_PUSH_TIME = 60
+    GITLAB_DOMAIN = 'wmf-gitlab3.vm.liw.fi'
+    GITLAB_PROJECT = 'liw'
+
+    def __init__(self, gitlab_token):
+        self._token = gitlab_token
+        self._tmpdir = tempfile.mkdtemp()
+        logging.info('Workspace: %s', self._tmpdir)
+
+    def get_routes(self):
+        return [
+            {
+                'method': 'POST',
+                'path': '/updaterepo',
+                'func': self._update_repo,
+                'scopes': ['update-repo'],
+            },
+        ]
+
+    def _update_repo(self):
+        spec = bottle.request.json
+        logging.info('Updating repository: %r', spec)
+
+        url = spec['git']
+        ref = spec['ref']
+        name = spec['gitlab']
+        dirname = os.path.join(self._tmpdir, name)
+
+        if self._clone(url, ref, dirname):
+            ok = self._remove(
+                self._token, self.GITLAB_DOMAIN, self.GITLAB_PROJECT, name)
+            if ok:
+                ok = self._push(
+                    dirname, self.GITLAB_DOMAIN, self.GITLAB_PROJECT, ref,
+                    name)
+                if ok:
+                    return 'Repository copied successfully'
+
+        logging.error('Something went wrong when copying repository')
+        return bottle.HTTPError(418)
+
+    def _clone(self, url, ref, dirname):
+        if os.path.exists(dirname):
+            logging.debug('Removing %s', dirname)
+            shutil.rmtree(dirname)
+        argv = ['git', 'clone', '-q', '-b', ref, url, dirname]
+        return runcmd('.', argv, self.MAX_CLONE_TIME)
+
+    def _remove(self, token, gitlab_domain, gitlab_project, name):
+        snippet = urllib.parse.quote('%s/%s' % (gitlab_project, name), safe='')
+        url = 'https://%s/api/v4/projects/%s' % (gitlab_domain, snippet)
+        argv = ['curl', '-HPRIVATE-TOKEN: %s' % token, '-X', 'DELETE', url]
+        return runcmd('.', argv, self.MAX_REMOVE_TIME)
+
+    def _push(self, dirname, gitlab_domain, gitlab_project, ref, name):
+        logging.info('Pushing %s to %s as %s', dirname, gitlab_domain, name)
+        url = 'ssh://git@%s/%s/%s.git' % (gitlab_domain, gitlab_project, name)
+        argv = ['git', 'push', url, '%s:master' % ref]
+        return runcmd(dirname, argv, self.MAX_PUSH_TIME)
+
+
+def runcmd(cwd, argv, timeout):
+    logging.info('Running command: %r', argv)
+    try:
+        p = subprocess.run(
+            argv, cwd=cwd, timeout=timeout, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+    except subprocess.TimeoutExpired:
+        logging.error('Command took too long (timeout %r)', timeout)
+        return False
+    except Exception as e:
+        logging.error('Error while running command: %s', str(e))
+        return False
+
+    if p.returncode != 0:
+        logging.error('Command failed: %r', argv)
+        logging.error('exit code: %d', p.returncode)
+        logging.error('stdout: %r', p.stdout)
+        logging.error('stderr: %r', p.stderr)
+        return False
+
+    logging.info('Command succeeded')
+    return True
+
+
+def setup_logging(log_filename):
+    logging.basicConfig(
+        filename=log_filename,
+        level=logging.DEBUG,
+        format='%(asctime)s %(levelname)s %(message)s')
+    logging.info('API server starting')
+
+
 def get_key_from_file(filename):
     with open(filename) as f:
         key_text = f.read()
     return Crypto.PublicKey.RSA.importKey(key_text)
 
-        
+
+def get_token_from_file(filename):
+    with open(filename) as f:
+        return f.read().strip()
+
+
 def main():
-    logging.basicConfig(
-        filename='api.log',
-        level=logging.DEBUG,
-        format='%(levelname)s %(message)s')
+    args = sys.argv[1:]
+    cmd = args.pop(0)
+    log_filename = args.pop(0)
+    pubkey_filename = args.pop(0)
+
+    setup_logging(log_filename)
 
-    filename = sys.argv[1]
-    key = get_key_from_file(filename)
+    if cmd == 'controller':
+        api = Controller()
+    elif cmd == 'vcsworker':
+        gitlab_token_filename = args.pop(0)
+        gitlab_token = get_token_from_file(gitlab_token_filename)
+        api = VCSWorker(gitlab_token)
+    else:
+        sys.exit('Unknown command %s' % cmd)
 
     app = bottle.Bottle()
-    api = Controller(app, key)
-    app.run(host='localhost', port=2222)
+    pubkey = get_key_from_file(pubkey_filename)
+    api.setup(app, pubkey)
+    app.run(host=HOST, port=PORT)
+
 
 main()