# Copyright 2017-2019 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import base64 import errno import json import os import random import signal import socket import sys import time import urllib import cliapp import requests import yaml from yarnutils import * srcdir = os.environ['SRCDIR'] datadir = os.environ['DATADIR'] V = Variables(datadir) def start_controller(): port = V['port'] = random_free_port() V['url'] = 'http://127.0.0.1:{}'.format(V['port']) filename = 'ick_controller.yaml' env = dict(os.environ) env['ICK_CONTROLLER_CONFIG'] = filename write_yaml(filename, { 'token-issuer': V['issuer'], 'token-audience': V['audience'], 'token-public-key': cat('token.key.pub'), 'log': [ { 'filename': 'ick_controller.log', }, ], 'statedir': V['statedir'], 'apt-server': 'localhost', 'artifact-store': V['artifact_store'], 'auth-url': V['auth_url'], 'notify-url': V['notify_url'], }) V['pid'] = gunicorn('ick_controller', 'app', port, env) def stop_controller(): if V['pid'] is not None: os.kill(int(V['pid']), signal.SIGTERM) def start_artifact_store(): port = V['bsport'] = random_free_port() V['bsurl'] = 'http://127.0.0.1:{}'.format(V['bsport']) filename = 'artifact_store.yaml' env = dict(os.environ) env['ARTIFACT_STORE_CONFIG'] = filename write_yaml(filename, { 'token-issuer': V['issuer'], 'token-audience': V['audience'], 'token-public-key': cat('token.key.pub'), 'log': [ { 'filename': 'artifact_store.log', }, ], 'blobdir': V['blobdir'], }) V['bspid'] = gunicorn('artifact_store', 'app', port, env) def stop_artifact_store(): if V['pid'] is not None: os.kill(int(V['bspid']), signal.SIGTERM) def write_yaml(filename, obj): yaml.safe_dump(obj, open(filename, 'w')) def gunicorn(module_name, var_name, port, env): log_filename = '{}.gunicorn.log'.format(module_name) pid_filename = '{}.pid'.format(module_name) argv = [ 'gunicorn3', '--daemon', '--bind', '127.0.0.1:{}'.format(port), '--log-file', log_filename, '--log-level', 'debug', '-p', pid_filename, '{}:{}'.format(module_name, var_name), ] cliapp.runcmd(argv, env=env) wait_for_port(port) return int(cat(pid_filename)) def random_free_port(): MAX = 1000 for i in range(MAX): port = random.randint(1025, 2**15-1) s = socket.socket() try: s.bind(('0.0.0.0', port)) except OSError as e: if e.errno == errno.EADDRINUSE: continue print('cannot find a random free port') raise s.close() break print('picked port', port) return port def wait_for_port(port): MAX = 5 t = time.time() while time.time() < t + MAX: try: s = socket.socket() s.connect(('127.0.0.1', port)) except socket.error: time.sleep(0.1) except OSError as e: raise else: return def unescape(s): t = '' while s: if s.startswith('\\n'): t += '\n' s = s[2:] else: t += s[0] s = s[1:] return t def write(filename, data): with open(filename, 'w') as f: f.write(data) def cat(filename): MAX_CAT_WAIT = 5 # in seconds t = time.time() while time.time() < t + MAX_CAT_WAIT: if os.path.exists(filename): return open(filename, 'r').read() def store_token(user, token): filename = '{}.jwt'.format(user) write(filename, token) def get_token(user): filename = '{}.jwt'.format(user) return cat(filename) def http(V, func, url, **kwargs): status, content_type, headers, body = func(url, **kwargs) V['status_code'] = status V['content_type'] = content_type V['headers'] = headers V['body'] = body def get(url, token): headers = { 'Authorization': 'Bearer {}'.format(token), } r = requests.get(url, headers=headers, verify=False) return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text def get_blob(url, token): headers = { 'Authorization': 'Bearer {}'.format(token), } r = requests.get(url, headers=headers, verify=False) return r.status_code, r.headers['Content-Type'], dict(r.headers), r.content def post(url, body, token): headers = { 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/json', } r = requests.post(url, headers=headers, data=body, verify=False) return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text def put(url, body, token): headers = { 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/json', } r = requests.put(url, headers=headers, data=body, verify=False) return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text def put_blob(url, body, token): headers = { 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/octet-stream', } r = requests.put(url, headers=headers, data=body, verify=False) return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text def delete(url, token): headers = { 'Authorization': 'Bearer {}'.format(token), } r = requests.delete(url, headers=headers, verify=False) return r.status_code, r.headers['Content-Type'], dict(r.headers), r.text def dict_diff(a, b): if not isinstance(a, dict): return 'first value is not a dict' if not isinstance(b, dict): return 'second value is not a dict' delta = [] for key in a: if key not in b: delta.append('second does not have key {}'.format(key)) elif isinstance(a[key], dict): delta2 = dict_diff(a[key], b[key]) if delta2 is not None: delta.append('key {}: dict values differ:'.format(key)) delta.append(delta2) elif isinstance(a[key], list): delta2 = list_diff(a[key], b[key]) if delta2 is not None: delta.append('key {}: list values differ:'.format(key)) delta.append(delta2) elif a[key] != b[key]: delta.append('key {}: values differ'.format(key)) delta.append(' first value : {!r}'.format(a[key])) delta.append(' second value: {!r}'.format(b[key])) for key in b: if key not in a: delta.append('first does not have key {}'.format(key)) if delta: return '\n'.join(delta) return None def list_diff(a, b): if not isinstance(a, list): return 'first value is not a list' if not isinstance(b, list): return 'second value is not a list' delta = [] for i in range(len(a)): if i >= len(b): delta.append('second list is shorter than first') break elif isinstance(a[i], dict): delta2 = dict_diff(a[i], b[i]) if delta2 is not None: delta.append('item {}: items are different dicts'.format(i)) delta.append(delta2) elif a[i] != b[i]: delta.append('item %d: values differ'.format(i)) delta.append(' first value : {!r}'.format(a[i])) delta.append(' second value: {!r}'.format(b[i])) if len(a) < len(b): delta.append('first list is shorter than second') if delta: return '\n'.join(delta) return None def encode_basename(basename): return urllib.quote(basename, safe='')