Add: some architecture documentation for ALPHA-1

4 files changed, 481 insertions, 0 deletions

diff --git a/arch/build.py b/arch/build.py
new file mode 100644
index 0000000..d60be05
--- /dev/null
+++ b/arch/build.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python2
+
+
+import os
+import sys
+import tempfile
+
+import cliapp
+
+
+def parse_markdown(f, tmpdir):
+    lines = f.readlines()
+    result = []
+    uml_filenames = []
+    while lines:
+        prefix, uml, lines = extract_uml(lines)
+        result.extend(prefix)
+        if uml:
+            filename = create_uml_file(tmpdir, uml)
+            uml_filenames.append(filename)
+            result.extend(include(uml, filename))
+    return result, uml_filenames
+
+
+def extract_uml(lines):
+    start = find_line(lines, '@startuml\n')
+    if start is None:
+        return lines, None, []
+
+    end = find_line(lines, '@enduml\n')
+    if end is None:
+        return lines, None, []
+
+    return lines[:start], lines[start:end+1], lines[end+1:]
+
+
+def find_line(lines, pattern):
+    for i, line in enumerate(lines):
+        if line.startswith(pattern):
+            return i
+    return None
+
+
+def create_uml_file(tmpdir, lines):
+    fd, filename = tempfile.mkstemp(dir=tmpdir)
+    os.write(fd, ''.join(lines))
+    os.close(fd)
+    return filename
+
+
+def include(uml, filename):
+    title = get_title(uml)
+    return [
+        '![{}]({}.png)\\ \n'.format(title, filename),
+        '\n',
+    ]
+
+
+def get_title(uml):
+    i = find_line(uml, 'title ')
+    if i is not None:
+        return uml[i].split(' ', 1)[1]
+
+
+def plantuml(filename):
+    cliapp.runcmd(['plantuml', '-tpng', '-output', '.', filename])
+
+
+def main():
+    tmpdir = sys.argv[1]
+    lines, filenames = parse_markdown(sys.stdin, tmpdir)
+    sys.stdout.write(''.join(lines))
+    for filename in filenames:
+        plantuml(filename)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/arch/build.sh b/arch/build.sh
new file mode 100755
index 0000000..034bd39
--- /dev/null
+++ b/arch/build.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -eu
+
+output="$(dirname "$1")/$(basename "$1" .mdwn)"
+
+tmp="$(mktemp -d)"
+
+clean()
+{
+    rm -rf "$tmp"
+}
+
+#trap clean EXIT
+
+cat -- "$@" | python2 ./build.py "$tmp" > "$tmp/foo.mdwn"
+pandoc --toc -V documentstyle:report --chapters -o "$output.pdf" "$tmp/foo.mdwn"
+pandoc --toc -o "$output.html" --number-sections --standalone -H ick2.css "$tmp/foo.mdwn"
diff --git a/arch/ick2-arch.mdwn b/arch/ick2-arch.mdwn
new file mode 100644
index 0000000..3234756
--- /dev/null
+++ b/arch/ick2-arch.mdwn
@@ -0,0 +1,306 @@
+---
+title: Ick2 architecture
+author: Lars Wirzenius
+date: work-in-progress for ALPHA-1
+...
+
+
+Introduction
+=============================================================================
+
+Ick2 consists of many micro services. This document describes how they
+are used individually and together.
+
+* The controller keeps track of projects, build pipelines, workers,
+  and the current state of each. It decides what build step is next,
+  and who should execute it.
+* Worker-manager represents a build host. It queries the controller
+  for work, and makes the build host (the actual worker) execute it,
+  and then reports results back to the worker.
+* The trigger service decides when a build should start.
+* The controller and trigger services provide an API. The identity
+  provider (IDP) takes care of the authentication of each API client,
+  and what privileges each should have. The API client authenticates
+  itself to the IDP, and receives an access token. The API provider
+  gets the token in each request, validates it, and inspects it to see
+  what the client is allowed to do.
+
+A major point of the IDP is to have just a single place where
+authentication and authorisation is configured.
+
+
+Individual API interactions
+=============================================================================
+
+This chapter covers interactions with individual APIs.
+
+
+Getting an access token
+-----------------------------------------------------------------------------
+
+The API client (user's command line tool, a putative web app, git
+server, worker-manager, etc) authenticates itself to the IDP, and if
+successful, gets back a signed JSON Web Token. It will include the
+token in all requests to all APIs so that the API provider will know
+what the client is allowed to do.
+
+The privileges for each API client are set by the sysadmin who
+installs the CI system, or a user who's been given IDP admin
+privileges by the sysadmin.
+
+@startuml
+hide footbox
+title Get an access token
+client -> IDP : GET /auth, with Basic Auth, over https
+IDP --> client : signed JWT token
+@enduml
+
+All API calls need a token. Getting a token happen the same way for
+every API client.
+
+
+Worker (worker-manager) registration
+-----------------------------------------------------------------------------
+
+The sysadmin arranges to start a worker-manager for every build host.
+They may run on the same host, or not: the Ick2 architecture doesn't
+really care. If they run on the same host, the worker manager will
+start a subprocess. If on different hosts, the subprocess will be
+started using ssh.
+
+The CI admin may define tags for each worker. Attributes may include
+things like whether the worker can be trusted with credentials for
+logging into other workers, or for retrieving source code from the git
+server. Workers may not override such tags. Workers may, however,
+provide other tags, to e.g., report their CPU architecture or Debian
+release. The controller will eventually be able to use the tags to
+choose which worker should execute which pipeline steps.
+
+@startuml
+hide footbox
+title Register worker
+worker_manager -> IDP : GET /auth, with Basic Auth, over https
+IDP --> worker_manager : token A
+worker_manager -> controller : POST /workers (token A)
+controller --> worker_manager : success
+@enduml
+
+The worker manager runs a very simple state machine.
+
+@startuml
+title Worker-manager state machine
+
+Querying : ask controller for work
+Running : run subprocess
+
+
+[*] -down-> Idle : start
+Idle -down-> Querying : short timeout has expired
+Querying -up-> Idle : nothing to do
+Querying --> Running : something to do
+
+Running --> Running : get output, report to controller
+Running --> Idle : subprocess finished, report to controller
+@enduml
+
+
+Add project to controller
+-----------------------------------------------------------------------------
+
+The CI admin (or a user authorised by the CI admin) adds projects to
+the controller to allow them to be built. This is done using an "CI
+administration application", which initially will be a command line
+tool, but may later become a web application as well. Either way, the
+controller provides API endpoints for this.
+
+@startuml
+hide footbox
+title Add project to controller
+
+adminapp -> IDP : GET /auth, with Basic Auth, over https
+IDP --> adminapp : token B
+adminapp -> controller : POST /projects (token B)
+controller --> adminapp : success or failure indication
+@enduml
+
+
+A full build
+=============================================================================
+
+Next we look at how the various compontens interact during a complete
+build, using two workers: one trusted with credentials, one doing the
+actual heavy work. We assume that workers have been registered and
+projects added.
+
+The sequence diagrams in this chapter have been split into stages, to
+make them easier to view and read. Each diagram after the first one
+continues where the previous one left off.
+
+Trigger build by pushing changes to git server
+-----------------------------------------------------------------------------
+
+@startuml
+hide footbox
+title Build triggered by git change
+
+developer -> gitano : git push
+
+gitano -> IDP : GET /auth, with Basic Auth, over https
+IDP --> gitano : token C
+gitano -> trigger : GET /git/website.git (token C)
+note right
+    Git server notifies
+    trigger service that
+    a git repo has changed
+end note
+
+|||
+
+trigger -> IDP : GET /auth, with Basic Auth, over https
+IDP --> trigger : token D
+trigger -> controller : GET /projects (token D)
+note right
+    trigger service queries
+    controller to get list
+    of all projects, so it
+    knows which builds to
+    start
+end note
+controller --> trigger : list of projects
+
+|||
+
+trigger -> controller : GET /projects/website (token D)
+note right
+    trigger service
+    gets project config
+    so it knows what
+    pipelines project has
+end note
+controller --> trigger : project description, incl. pipelines
+
+|||
+
+trigger -> controller : POST /projects/website/pipelines/getsource/+start (token D)
+@enduml
+
+The first pipeline has now been started by the trigger service.
+
+
+Pipeline 1: get sources
+-----------------------------------------------------------------------------
+
+The first pipeline uses the trusted worker to fetch source code from
+the git server (we assume that requires credentials), and push them
+to the powerful worker.
+
+@startuml
+hide footbox
+title Build pipeline: get source
+
+trusty -> IDP : GET /auth, with Basic Auth, over https
+IDP --> trusty : token E
+
+|||
+
+trusty -> controller : GET /worker/trusty (token E)
+controller --> trusty : "clone website source into workspace"
+trusty -> gitano : git clone
+trusty -> controller : POST /worker/trusty, exit=0 (token E)
+
+|||
+
+trusty -> controller : GET /worker/trusty (token E)
+controller --> trusty : "rsync to beefy"
+trusty -> beefy      : rsync
+trusty -> controller : POST /worker/trusty, exit=0 (token E)
+
+|||
+
+trusty -> controller : GET /worker/trusty (token E)
+controller -> trusty  : "notify trigger service pipeline is finished"
+trusty -> controller  : POST /worker/trusty, exit=0 (token E)
+note right
+    note that we need to tell controller this last step is finished
+    **before** telling the trigger service the pipeline is finished
+end note
+trusty -> trigger     : GET /pipelines/website/getsource (token E)
+trigger -> controller : GET /projects/website/pipelines/getsource (token D)
+note right
+    Trigger service checks
+    status of pipeline
+end note
+trigger -> controller : POST /projects/website/pipelines/ikiwiki/+start (token D)
+@enduml
+
+The first pipeline finished, and the website building can start.
+That's the second pipeline, which has just been started.
+
+
+Pipeline 2: Build static web site
+-----------------------------------------------------------------------------
+
+The second pipeline runs on the beefy worker. The source is already
+there and it just needs to perform the build.
+
+@startuml
+hide footbox
+title Build static website
+
+beefy -> IDP : GET /auth, with Basic Auth, over https
+IDP --> beefy : token F
+beefy -> controller : GET /worker/beefy (token F)
+controller -> beefy : "build static website"
+beefy -> beefy : run ikiwiki to build site
+beefy -> controller : POST /worker/beefy, exit=0 (token F)
+
+|||
+
+beefy -> controller : GET /worker/beefy (token F)
+controller -> beefy  : "notify trigger service pipeline is finished"
+beefy -> controller  : POST /worker/beefy, exit=0 (token F)
+beefy -> trigger     : GET /pipelines/website/ikiwiki (token F)
+trigger -> controller : GET /projects/website/pipelines/ikiwiki (token D)
+trigger -> controller : POST /projects/website/pipelines/publish/+start (token D)
+
+@enduml
+
+At the end of the second pipeline, we start the third one.
+
+Pipeline 3: Publish web site to web server
+-----------------------------------------------------------------------------
+
+The third pipeline copies the built static website from the beefy
+worker, via the trusted worker, to the actual web server.
+
+@startuml
+hide footbox
+title Copy built site from beefy to web server
+
+trusty -> controller : GET /worker/trusty (token E)
+controller -> trusty : "rsync workspace from beefy"
+trusty -> beefy      : rsync
+beefy --> trusty     : workspace updates
+trusty -> controller : POST /worker/trusty, exit=0 (token E)
+
+|||
+
+trusty -> controller : GET /worker/trusty (token E)
+controller -> trusty : "rsync static website to web server"
+trusty -> webserver  : rsync
+trusty -> controller : POST /worker/trusty, exit=0 (token E)
+
+|||
+
+trusty -> controller : GET /worker/trusty (token E)
+controller --> trusty : "notify trigger service pipeline is finished"
+trusty -> controller  : POST /worker/trusty, exit=0 (token E)
+trusty -> trigger     : GET /pipelines/website/publish (token E)
+trigger -> controller : GET /projects/website/pipelines/ikiwiki (token D)
+note right
+    There are no further pipelines.
+end note
+
+@enduml
+
+The website is now built and published.
diff --git a/arch/ick2.css b/arch/ick2.css
new file mode 100644
index 0000000..78cd374
--- /dev/null
+++ b/arch/ick2.css
@@ -0,0 +1,79 @@
+<style>
+html {
+    background: white;
+    font-family: serif;
+    margin-left: 3em;
+    margin-right: 2em;
+    margin-top: 2em;
+}
+
+form#searchform {
+    font-family: monospace;
+    text-align: right;
+}
+
+div.actions {
+    font-family: monospace;
+    text-align: right;
+}
+
+div.actions ul, div.actions li {
+    display: inline;
+}
+
+div.pageheader {
+    font-family: monospace;
+    margin-bottom: 2em;
+}
+
+div.pageheader span.title {
+    display: block;
+    font-size: 200%;
+    font-weight: bold;
+    font-family: sans-serif;
+    margin-top: 0.5em;
+}
+
+div#pagebody {
+}
+
+div.pagefooter {
+    font-family: monospace;
+    margin-top: 3em;
+}
+
+div#pagebody {
+}
+
+div#TOC ul {
+    list-style: none;
+}
+
+h1, h2, h3, h4, h5, h6 {
+    font-family: sans-serif;
+    font-weight: bold;
+    margin-top: 2em;
+}
+
+h1 {
+    font-size: 150%;
+}
+
+h2 {
+    font-size: 120%;
+}
+
+h3 {
+    font-size: 100%;
+}
+
+ul li, ol li {
+    margin-top: 0.5em;
+    margin-bottom: 0.5em;
+}
+
+pre {
+    margin-left: 4em;
+}
+
+</style>