diff options
44 files changed, 1548 insertions, 88 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..246e261 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +subplot.html +subplot.pdf +test.cfg +test.log +test.md +test.py +qemu.out +ssh.config @@ -2,7 +2,7 @@ README for debian-ansible ============================================================================= This is a small collection of Ansible roles for Debian targets. These -roles care chosen based on generic usefulness, and are meant to be +roles are chosen based on generic usefulness, and are meant to be highly cohesive, loosely coupled, and parameterised as needed. Legalese @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Run the automated tests for the project. + +set -eu -o pipefail + +cat_with_sep() { + for x in "$@"; do + cat "$x" + echo + done +} + +hideok=chronic +if [ "$#" -gt 0 ]; then + case "$1" in + verbose | -v | --verbose) + hideok= + shift 1 + ;; + esac +fi + +dir="$(mktemp -d -p .)" + +trap 'rm -rf "$dir"' EXIT + +rm -f test.log test.py +cp subplot.subplot subplot.md "$dir" +cat_with_sep subplot.md roles/*/subplot.md >"$dir/subplot.md" +cat_with_sep subplot/*.py roles/*/subplot.py >"$dir/subplot.py" +cat_with_sep subplot/*.yaml roles/*/subplot.yaml >"$dir/subplot.yaml" + +( + set -eu -o pipefail + cd "$dir" + subplot docgen subplot.subplot -o ../subplot.pdf + subplot docgen subplot.subplot -o ../subplot.html + subplot codegen subplot.subplot -o ../test.py +) + +# Fix private key permissions. git doesn't preserve them. +chmod 0600 ssh/id + +# Create configuration for the test VMs used by the test suite. +cat >test.cfg <<EOF +name: debian-ansible-test +base_image: "$HOME/tmp/debian.qcow2" +username: debian +cpus: 2 +memory: 1024 +EOF + +$hideok python3 test.py --log test.log --env ANSIBLE_STDOUT_CALLBACK=yaml "$@" + +echo "Everything seems to be in order." diff --git a/roles/apache_server/subplot.md b/roles/apache_server/subplot.md new file mode 100644 index 0000000..79fa071 --- /dev/null +++ b/roles/apache_server/subplot.md @@ -0,0 +1,5 @@ +# Role `apache_server` – set up a web server using Apache + +This role sets up a host to be a web server using the Apache +software. However, as I don't plan on working on the role, I haven't +made any acceptance criteria for it yet. diff --git a/roles/apache_server/tasks/main.yml b/roles/apache_server/tasks/main.yml index e01d01e..5509818 100644 --- a/roles/apache_server/tasks/main.yml +++ b/roles/apache_server/tasks/main.yml @@ -67,6 +67,7 @@ name: "{{ item }}" with_items: - ssl + - rewrite - name: create dirs for static site contents file: @@ -101,6 +102,12 @@ notify: - restart apache +- name: set default charset to utf8 + copy: + content: | + AddDefaultCharset UTF-8 + dest: /etc/apache2/conf-available/charset.conf + - name: "install htpasswd files" copy: content: "{{ item.htpasswd }}" diff --git a/roles/apache_server/templates/deploy_static_site_certs b/roles/apache_server/templates/deploy_static_site_certs index 4933d56..1040144 100644 --- a/roles/apache_server/templates/deploy_static_site_certs +++ b/roles/apache_server/templates/deploy_static_site_certs @@ -32,10 +32,12 @@ run_certbot() } -systemctl stop apache2 +systemctl stop apache2 || true + for list in /etc/letsencrypt/*.list do certname="$(basename "$list" .list)" run_certbot "$list" "$certname" || true done -systemctl start apache2 +systemctl start apache2 || true + diff --git a/roles/apache_server/templates/virtualhost.conf.tmpl b/roles/apache_server/templates/virtualhost.conf.tmpl index 8d069ce..1e14db5 100644 --- a/roles/apache_server/templates/virtualhost.conf.tmpl +++ b/roles/apache_server/templates/virtualhost.conf.tmpl @@ -8,6 +8,10 @@ ErrorLog /var/log/apache2/{{ item.domain }}/error.log CustomLog /var/log/apache2/{{ item.domain }}/access.log combined <Directory /srv/http/{{ item.domain }}> +{% if item.redirect|default(false) %} + Redirect permanent / "https://{{ item.redirect }}/" + Require all granted +{% else %} {% if item.letsencrypt|default(false) %} Redirect permanent / "https://{{ item.domain }}/" Require all granted @@ -23,6 +27,7 @@ Require all granted {% endif %} {% endif %} +{% endif %} </Directory> Alias /.well-known/ /srv/letsencrypt/{{ item.domain }}/ @@ -45,6 +50,10 @@ CustomLog /var/log/apache2/{{ item.domain }}/access.log combined <Directory /srv/http/{{ item.domain }}> Options +SymlinksIfOwnerMatch +Indexes +MultiViews +{% if item.redirect|default(false) %} + Redirect permanent / "https://{{ item.redirect }}/" + Require all granted +{% else %} {% if item.htpasswd is defined %} AuthType Basic AuthName "{{ item.htpasswd_name }}" @@ -54,6 +63,7 @@ AllowOverride AuthConfig Require all granted {% endif %} +{% endif %} </Directory> SSLEngine on diff --git a/roles/gitano_server/subplot.md b/roles/gitano_server/subplot.md new file mode 100644 index 0000000..1fc2ea0 --- /dev/null +++ b/roles/gitano_server/subplot.md @@ -0,0 +1,7 @@ +# Role `gitano_server` – set up a Gitano git server + +This role sets up a host to be a git server using the [Gitano][] +software. However, as I don't plan on working on the role, I haven't +made any acceptance criteria for it yet. + +[Gitano]: https://www.gitano.org.uk/ diff --git a/roles/gitano_server/tasks/main.yml b/roles/gitano_server/tasks/main.yml index 08486fa..b25d142 100644 --- a/roles/gitano_server/tasks/main.yml +++ b/roles/gitano_server/tasks/main.yml @@ -1,3 +1,6 @@ -- include: gitano.yml -- include: git-daemon.yml -- include: cgit.yml +- ansible.builtin.import_tasks: + file: gitano.yml +- ansible.builtin.import_tasks: + file: git-daemon.yml +- ansible.builtin.import_tasks: + file: cgit.yml diff --git a/roles/haproxy/subplot.md b/roles/haproxy/subplot.md new file mode 100644 index 0000000..660cfd3 --- /dev/null +++ b/roles/haproxy/subplot.md @@ -0,0 +1,5 @@ +# Role `haproxy` – set up a web proxy using haproxy + +This role sets up a host to be a web proxy using the haproxy +software. However, as I don't plan on working on the role, I haven't +made any acceptance criteria for it yet. diff --git a/roles/radicle_node/README.md b/roles/radicle_node/README.md new file mode 100644 index 0000000..b653160 --- /dev/null +++ b/roles/radicle_node/README.md @@ -0,0 +1,2 @@ +This role, `radicle_node`, sets up a Debian system to be a +[Radicle](https://radicle.xyz/) node. diff --git a/roles/radicle_node/files/rad-config-pin b/roles/radicle_node/files/rad-config-pin new file mode 100644 index 0000000..0e40f00 --- /dev/null +++ b/roles/radicle_node/files/rad-config-pin @@ -0,0 +1,23 @@ +#!/usr/bin/python3 + +import json, os, subprocess, sys + +rid = sys.argv[1] + +p = subprocess.run(["rad", "config", "show"], check=True, capture_output=True) +if p.returncode != 0: + sys.exit("rad config show failed") +config = json.loads(p.stdout.decode()) + +config["web"]["pinned"]["repositories"].append(rid) + +p = subprocess.run(["rad", "self", "--home"], check=True, capture_output=True) +if p.returncode != 0: + sys.exit("rad self --home failed") + +home = p.stdout.decode().strip() +filename = os.path.join(home, "config.json") +if os.path.exists(filename): + os.rename(filename, filename + ".bak") +with open(filename, "w") as f: + f.write(json.dumps(config, indent=4)) diff --git a/roles/radicle_node/files/rad-config-update b/roles/radicle_node/files/rad-config-update new file mode 100644 index 0000000..40dd1a9 --- /dev/null +++ b/roles/radicle_node/files/rad-config-update @@ -0,0 +1,34 @@ +#!/usr/bin/python3 + +import json, os, subprocess, sys + +alias = sys.argv[1] +ext = sys.argv[2] +policy = sys.argv[3] +scope = sys.argv[4] +peer = sys.argv[5] + +p = subprocess.run(["rad", "config", "show"], capture_output=True) +if p.returncode != 0: + sys.exit("rad config show failed") +config = json.loads(p.stdout.decode()) + +config["node"]["alias"] = alias +config["node"]["externalAddresses"] = [ext] +config["node"]["policy"] = policy +config["node"]["scope"] = scope + +nodes = config["node"]["connect"] +if peer not in nodes: + nodes.append(peer) + +p = subprocess.run(["rad", "self", "--home"], check=True, capture_output=True) +if p.returncode != 0: + sys.exit("rad self --home failed") + +home = p.stdout.decode().strip() +filename = os.path.join(home, "config.json") +if os.path.exists(filename): + os.rename(filename, filename + ".bak") +with open(filename, "w") as f: + f.write(json.dumps(config, indent=4)) diff --git a/roles/radicle_node/tasks/main.yml b/roles/radicle_node/tasks/main.yml new file mode 100644 index 0000000..8e04a83 --- /dev/null +++ b/roles/radicle_node/tasks/main.yml @@ -0,0 +1,242 @@ +- name: "check radicle_node_version" + shell: | + [ "{{ radicle_node_version }}" = "1" ] || \ + (echo "Unexpected version {{ radicle_node_version }}" 1>&2; exit 1) + +- name: "check that radicle_node_key is set" + shell: | + echo radicle_node_key Ansible variable is not set + exit 1 + when: radicle_node_key is not defined + +- name: "check that radicle_node_key_pub is set" + shell: | + echo radicle_node_key_pub Ansible variable is not set + exit 1 + when: radicle_node_key_pub is not defined + +- name: "install important additional packages for Radicle" + apt: + name: + # For the Radicle installer + - curl + + # Radicle is built on git. + - git + + # Rsync for backups. + - rsync + + # Web server for the web UI. + - caddy + + # Radicle components. + - radicle + - radicle-ci-broker + - radicle-native-ci + +- name: "stop Radicle node if it's running" + shell: | + systemctl stop radicle-node || true + +- name: "stop Radicle CI broker if it's running" + shell: | + systemctl stop radicle-ci-broker || true + +- name: "configure git for _rad user" + shell: | + sudo -u _rad git config --global user.name "_rad" + sudo -u _rad git config --global user.email "liw@liw.fi" + +- name: "create directory for Radicle for the _rad user" + file: + state: directory + path: /home/_rad/.radicle + owner: _rad + group: _rad + mode: 0755 + +- name: "create directory for web pages" + file: + state: directory + path: /srv/http + owner: _rad + group: _rad + mode: 0755 + +- name: "create directory for Radicle backup" + when: radicle_node_backup is defined + file: + state: directory + path: radicle-backup + owner: root + group: root + mode: 0755 + +- name: "restore from backup (step 1 or 2)" + when: radicle_node_backup is defined + synchronize: + src: "{{ radicle_node_backup }}/." + dest: radicle-backup/. + group: no + owner: no + +- name: "restore from backup (step 2 or 2)" + when: radicle_node_backup is defined + shell: | + find radicle-backup -name control.sock -delete + rsync -a --del radicle-backup/home/_rad/.radicle/. /home/_rad/.radicle/. + rsync -a --del radicle-backup/srv/http/. /srv/http/. + chown -R _rad:_rad /home/_rad/.radicle/. /srv/http/. + +- name: "create directory for Radicle keys" + file: + state: directory + path: /home/_rad/.radicle/keys + owner: _rad + group: _rad + mode: 0755 + +- name: "install Radicle private key" + copy: + content: "{{ radicle_node_key }}" + dest: /home/_rad/.radicle/keys/radicle + owner: _rad + group: _rad + mode: 0600 + +- name: "install Radicle public key" + copy: + content: "{{ radicle_node_key_pub }}" + dest: /home/_rad/.radicle/keys/radicle.pub + owner: _rad + group: _rad + mode: 0644 + +- name: "install systemd unit for Radicle node" + template: + src: radicle-node.service.j2 + dest: /lib/systemd/system/radicle-node.service + +- name: "init Radicle node config" + shell: | + if [ ! -e /home/_rad/.radicle/config.json ]; then + sudo -u _rad -i rad config init --alias "{{ radicle_node_domain_name }}" + fi + +- name: "(re)start systemd unit for Radicle node" + systemd: + name: radicle-node + state: restarted + masked: no + enabled: yes + daemon_reload: yes + +- name: "install script to add update Radicle config file" + when: radicle_node_connections is defined + copy: + src: rad-config-update + dest: /home/_rad/rad-config-update + owner: _rad + group: _rad + mode: 0755 + +- name: "connect to other Radicle nodes" + when: radicle_node_connections is defined + with_items: "{{ radicle_node_connections }}" + shell: | + sudo -u _rad -i ./rad-config-update \ + "{{ radicle_node_domain_name }}" \ + "{{ radicle_node_domain_name }}:8776" \ + "{{ radicle_node_policy }}" \ + "{{ radicle_node_scope }}" \ + "{{ item.nid }}@{{ item.host }}:{{ item.port }}" + +- name: "install script to add update Radicle repository pinning" + when: radicle_node_repositories is defined + copy: + src: rad-config-pin + dest: /home/_rad/rad-config-pin + owner: _rad + group: _rad + mode: 0755 + +- name: "seed Radicle repositories" + when: radicle_node_repositories is defined + with_items: "{{ radicle_node_repositories }}" + shell: | + sudo -u _rad rad seed "{{ item.rid }}" + sudo -u _rad -i ./rad-config-pin "{{ item.rid }}" + +- name: "install Caddy configuation file" + template: + src: Caddyfile.j2 + dest: /etc/caddy/Caddyfile + +- name: "create directory for CI logs" + file: + state: directory + path: /srv/http + owner: _rad + group: _rad + +- name: "restart Caddy" + systemd: + name: caddy + state: restarted + masked: no + enabled: yes + daemon_reload: yes + +- name: "install systemd unit for Radicle HTTPD" + template: + src: radicle-httpd.service.j2 + dest: /lib/systemd/system/radicle-httpd.service + +- name: "enable systemd unit for Radicle HTTPD" + systemd: + name: radicle-httpd + state: restarted + masked: no + enabled: yes + daemon_reload: yes + +- name: "install Radicle CI broker config" + copy: + content: | + {{ radicle_node_ci_broker_config }} + dest: /home/_rad/ci-broker.yaml + owner: _rad + group: _rad + mode: 0644 + +- name: "create state directory for Radicle native CI" + file: + state: directory + path: /home/_rad/native-ci.state + owner: _rad + group: _rad + mode: 0755 + +- name: "install Radicle native CI config" + copy: + content: | + state: /srv/http + log: /home/_rad/native-ci.log + dest: /home/_rad/native-ci.yaml + owner: _rad + group: _rad + mode: 0644 + +- name: "install systemd unit for Radicle CI broker" + template: + src: radicle-ci-broker.service.j2 + dest: /lib/systemd/system/radicle-ci-broker.service + +- name: "enable systemd unit for Radicle CI broker" + systemd: + name: radicle-ci-broker + state: restarted + masked: no + enabled: yes + daemon_reload: yes diff --git a/roles/radicle_node/templates/Caddyfile.j2 b/roles/radicle_node/templates/Caddyfile.j2 new file mode 100644 index 0000000..1954b4d --- /dev/null +++ b/roles/radicle_node/templates/Caddyfile.j2 @@ -0,0 +1,14 @@ +:80 { + root * /usr/share/caddy +} +{{ radicle_node_domain_name }}:443 { + reverse_proxy 127.0.0.1:8080 +} +{{ radicle_node_ci_domain_name }}:443 { + root * /srv/http/ + file_server browse +} +{{ radicle_node_wumpus_domain_name }}:443 { + root * /srv/wumpus/ + file_server browse +} diff --git a/roles/radicle_node/templates/radicle-ci-broker.service.j2 b/roles/radicle_node/templates/radicle-ci-broker.service.j2 new file mode 100644 index 0000000..239ccc5 --- /dev/null +++ b/roles/radicle_node/templates/radicle-ci-broker.service.j2 @@ -0,0 +1,18 @@ +[Unit] +After=radicle-node.service +Description=Radicle CI broker + +[Service] +Type=simple +Environment=RAD_HOME=/home/_rad/.radicle +Environment=PATH=/home/_rad/.cargo/bin:/bin:/usr/bin:/sbin:/usr/sbin +Environment=RUST_LOG=info +ExecStart=/bin/ci-broker /home/_rad/ci-broker.yaml +KillMode=control-group +Restart=always +RestartSec=1 +User=_rad +Group=_rad + +[Install] +WantedBy=default.target diff --git a/roles/radicle_node/templates/radicle-httpd.service.j2 b/roles/radicle_node/templates/radicle-httpd.service.j2 new file mode 100644 index 0000000..32b2ecf --- /dev/null +++ b/roles/radicle_node/templates/radicle-httpd.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Radicle HTTP Daemon +After=network.target network-online.target +Requires=network-online.target + +[Service] +User=_rad +Group=_rad +ExecStart=/usr/bin/radicle-httpd --listen 127.0.0.1:8080 +Environment=RAD_HOME=/home/_rad/.radicle RUST_BACKTRACE=1 RUST_LOG=info +KillMode=process +Restart=always +RestartSec=1 + +[Install] +WantedBy=multi-user.target diff --git a/roles/radicle_node/templates/radicle-node.service.j2 b/roles/radicle_node/templates/radicle-node.service.j2 new file mode 100644 index 0000000..ae2af8c --- /dev/null +++ b/roles/radicle_node/templates/radicle-node.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Radicle Node +After=network.target network-online.target +Requires=network-online.target + +[Service] +User=_rad +Group=_rad +ExecStart=/usr/bin/radicle-node --listen 0.0.0.0:8776 --force +Environment=RAD_HOME=/home/_rad/.radicle RUST_BACKTRACE=1 RUST_LOG=info +KillMode=process +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/roles/sane_debian_system/defaults/main.yml b/roles/sane_debian_system/defaults/main.yml index 9e2ca84..d896b75 100644 --- a/roles/sane_debian_system/defaults/main.yml +++ b/roles/sane_debian_system/defaults/main.yml @@ -1,15 +1,20 @@ # These are the variables expected by this role. -# The desired hostname. Default is empty, which means hostname won't -# be set. -hostname: "" +# Playbook should set this to the version of this role it expects to +# use. Defaults to the inventory hostname. +sane_debian_system_version: "{{ inventory_hostname }}" + + +# The desired hostname. There is no no default, which means hostname +# won't be set. +sane_debian_system_hostname: "" # The Debian release code name to use. -debian_codename: +sane_debian_system_codename: # Default Debian mirror to use. Default should work everywhere, but if # needed, pick a faster mirror, perhaps a local one. -debian_mirror: deb.debian.org +sane_debian_system_mirror: deb.debian.org # A list of extra APT repositories to add. Each list entry should be a @@ -18,13 +23,13 @@ debian_mirror: deb.debian.org # keys are "signing_key", the public key of the archive signing key, # and "keyring_packge", which contains the .deb package with the # archive signing key. -sources_lists: [] +sane_debian_system_sources_lists: [] # Locales that should be generated. This should be a list of name, such as # fi_FI.UTF-8. -locales: [] +sane_debian_system_locales: [] # Default time zone. -timezone: UTC +sane_debian_system_timezone: UTC diff --git a/roles/sane_debian_system/subplot.md b/roles/sane_debian_system/subplot.md new file mode 100644 index 0000000..be05984 --- /dev/null +++ b/roles/sane_debian_system/subplot.md @@ -0,0 +1,66 @@ +# Role `sane_debian_system` – set up a manageable Debian system + +This role sets up a Debian system so that it can be managed with +Ansible in a reasonable way. + +## Version history + +### Version 2 + +* `sane_debian_hostname` defaults to the inventory hostname. This + means it's not necessary to set it if the default is sufficient. + +## Minimally sane Debian system + +~~~scenario +given a host running Debian +when I use role sane_debian_system +and I use variables from sane1.yml +and I run the playbook +then the host has the sudo package installed +and the host has the apt-transport-https package installed +and the host has the locales package installed +and the host has the systemd-timesyncd package installed +and the host has an empty /etc/apt/sources.list.d directory +and the host has hostname saneone +~~~ + +~~~{#sane1.yml .file .yaml} +ansible_python_interpreter: /usr/bin/python3 + +sane_debian_system_version: 2 +sane_debian_system_codename: bullseye +sane_debian_system_hostname: saneone +~~~ + +## Uses inventory hostname by default + +~~~scenario +given a host running Debian +when I use role sane_debian_system +and I use variables from sane-without-hostname.yml +and I run the playbook +then the host has the sudo package installed +and the host has hostname debian-ansible-test +~~~ + +~~~{#sane-without-hostname.yml .file .yaml} +sane_debian_system_version: 2 + +sane_debian_system_codename: bullseye +~~~ + +## Checks that debian codename is set + +~~~scenario +given a host running Debian +when I use role sane_debian_system +and I use variables from sane2.yml +and I try to run the playbook +then the command fails +and stdout contains "sane_debian_system_codename" +~~~ + +~~~{#sane2.yml .file .yaml} +sane_debian_system_version: 2 +~~~ diff --git a/roles/sane_debian_system/subplot.py b/roles/sane_debian_system/subplot.py new file mode 100644 index 0000000..f45eb2b --- /dev/null +++ b/roles/sane_debian_system/subplot.py @@ -0,0 +1,48 @@ +import logging + + +def host_has_package_installed(ctx, package=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["dpkg", "--status", package]) + assert_eq(exit, 0) + installed = False + for line in output.decode("UTF8").splitlines(): + if line.startswith("Status:") and " installed" in line: + installed = True + break + assert installed + + +def host_directory_is_empty(ctx, pathname=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["find", "/etc/apt/sources.list.d"]) + assert_eq(exit, 0) + for line in output.decode("UTF8").splitlines(): + assert "/etc/apt/sources.list.d/" not in line + + +def host_hostname_is(ctx, hostname=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["hostname"]) + assert_eq(exit, 0) + actual = output.decode("UTF8").splitlines()[-1] + assert_eq(actual, hostname) + + +def host_hostname_has_address(ctx, hostname=None, addr=None): + assert_eq = globals()["assert_eq"] + + logging.debug(f"host_hostname_has_address:") + qemu = ctx["qemu"] + output, exit = qemu.ssh(["cat", "/etc/hosts"]) + assert_eq(exit, 0) + logging.debug(f" /etc/hosts: {output!r}") + actual = output.decode("UTF8") + logging.debug(f" /etc/hosts: {actual!r}") + wordlines = [line.split() for line in actual.splitlines()] + matches = [words for words in wordlines if len(words) == 2 and words[1] == hostname] + logging.debug(f" matches: {matches!r}") + assert_eq(matches, [[addr, hostname]]) diff --git a/roles/sane_debian_system/subplot.yaml b/roles/sane_debian_system/subplot.yaml new file mode 100644 index 0000000..b7f03cb --- /dev/null +++ b/roles/sane_debian_system/subplot.yaml @@ -0,0 +1,19 @@ +- then: the host has the {package} package installed + impl: + python: + function: host_has_package_installed + +- then: the host has an empty {pathname} directory + impl: + python: + function: host_directory_is_empty + +- then: the host has hostname {hostname} + impl: + python: + function: host_hostname_is + +- then: the host has {hostname} in /etc/hosts for {addr} + impl: + python: + function: host_hostname_has_address diff --git a/roles/sane_debian_system/tasks/apt.yml b/roles/sane_debian_system/tasks/apt.yml index 84c6420..0da3332 100644 --- a/roles/sane_debian_system/tasks/apt.yml +++ b/roles/sane_debian_system/tasks/apt.yml @@ -1,9 +1,9 @@ # Safety check: make sure debian_codename is set. -- name: check that debian_codename is set +- name: check that sane_debian_system_codename is set shell: | - if [ "{{ debian_codename }}" = "" ] + if [ "{{ sane_debian_system_codename }}" = "" ] then - echo "You MUST set debian_codename" 1>&2 + echo "You MUST set sane_debian_system_codename" 1>&2 exit 1 fi @@ -11,8 +11,12 @@ # First update package lists. The ones that come with the image may be # badly out of date. # -# Ignore any error here so that later tasks can fix things such as a badly -# formed sources.list. +# Use shell to run apt-get, rather than the Ansible apt module, so +# that we can pass in the --allow-releaseinfo--change option. +- name: update package lists + shell: | + apt-get update --allow-releaseinfo-change + - name: update package lists ignore_errors: yes apt: @@ -24,11 +28,7 @@ # will still work. apt-transport-https is in the main Debian archive, # and we assume those are in the sources.list that come with the # image. -# -# Ignore any error here so that later tasks can fix things such as a badly -# formed sources.list. - name: install apt-transport-https - ignore_errors: yes apt: name: apt-transport-https @@ -37,26 +37,45 @@ src: sources.list.j2 dest: /etc/apt/sources.list +- name: "update package lists" + apt: + update_cache: yes + +- name: install necessary tools + apt: + name: + - sudo + +- name: "allow root to use sudo" + copy: + content: | + root ALL=(ALL:ALL) NOPASSWD: ALL + dest: /etc/sudoers.d/root + mode: 0600 + - name: additional sources.list.d/* - with_items: "{{ sources_lists }}" + with_items: "{{ sane_debian_system_sources_lists }}" apt_repository: repo: "{{ item.repo }}" update_cache: no - name: add archive signing keys - with_items: "{{ sources_lists }}" - apt_key: - data: "{{ item.signing_key }}" - state: present + with_items: "{{ sane_debian_system_sources_lists }}" + shell: | + key="{{ item.signing_key }}" + sum="$(echo -n "$key" | sha1sum | awk '{ print $1 }')" + echo "$key" > "/etc/apt/trusted.gpg.d/$sum.asc" when: item.signing_key is defined +# Use shell to run apt-get to update package lists so that we can pass +# in the --allow-releaseinfo--change option. - name: update package lists - apt: - update_cache: yes - cache_valid_time: 0 + shell: | + apt-get update --allow-releaseinfo-change + - name: add archive keyrings - with_items: "{{ sources_lists }}" + with_items: "{{ sane_debian_system_sources_lists }}" apt: name: "{{ item.keyring_package }}" when: item.keyring_package is defined diff --git a/roles/sane_debian_system/tasks/env.yml b/roles/sane_debian_system/tasks/env.yml index c62fca4..eedd864 100644 --- a/roles/sane_debian_system/tasks/env.yml +++ b/roles/sane_debian_system/tasks/env.yml @@ -2,35 +2,38 @@ apt: name: dbus -- name: set /etc/hostname - copy: - content: "{{ hostname }}" - dest: /etc/hostname - owner: root - group: root - mode: 0444 - when: hostname is defined +- name: "start dbus" + systemd: + name: dbus + daemon_reload: yes + enabled: yes + state: started -- name: add hostname to /etc/hosts - lineinfile: - dest: /etc/hosts - regexp: '^127\.0\.1\.1 ' - line: "127.0.1.1 {{ hostname }}" - when: hostname is defined +- name: set /etc/hostname + hostname: + name: "{{ sane_debian_system_hostname }}" + when: sane_debian_system_hostname != "" - name: set timezone timezone: - name: "{{ timezone }}" + name: "{{ sane_debian_system_timezone }}" - name: install environment packages apt: state: present name: - locales - - ntp + +- name: install systemd-timesyncd or ntp + shell: | + if apt-cache show systemd-timesyncd > /dev/null; then + DEBIAN_FRONTEND=noninteractife apt-get install -y systemd-timesyncd + else + DEBIAN_FRONTEND=noninteractife apt-get install -y ntp + fi - name: generate locales locale_gen: name: "{{ item }}" state: present - with_items: "{{ locales }}" + with_items: "{{ sane_debian_system_locales }}" diff --git a/roles/sane_debian_system/tasks/main.yml b/roles/sane_debian_system/tasks/main.yml index dcb3b60..bc8c6d3 100644 --- a/roles/sane_debian_system/tasks/main.yml +++ b/roles/sane_debian_system/tasks/main.yml @@ -1,2 +1,10 @@ -- include: apt.yml -- include: env.yml +- name: "sane_debian_system_version" + shell: | + [ "{{ sane_debian_system_version }}" = "2" ] || \ + (echo "Unexpected version {{ sane_debian_system_version }}" 1>&2; exit 1) + +- ansible.builtin.import_tasks: + file: apt.yml + +- ansible.builtin.import_tasks: + file: env.yml diff --git a/roles/sane_debian_system/templates/sources.list.j2 b/roles/sane_debian_system/templates/sources.list.j2 index 65e8706..897e2ce 100644 --- a/roles/sane_debian_system/templates/sources.list.j2 +++ b/roles/sane_debian_system/templates/sources.list.j2 @@ -1,12 +1 @@ -deb http://{{ debian_mirror }}/debian {{ debian_codename }} main - -{% if debian_codename != 'unstable' %} -deb http://security.debian.org/ {{ debian_codename }}/updates main -{% endif %} - -{% if debian_codename != 'buster' %} -{% if debian_codename != 'unstable' %} -deb http://{{ debian_mirror }}/debian {{ debian_codename }}-updates main -deb http://{{ debian_mirror }}/debian {{ debian_codename }}-backports main -{% endif %} -{% endif %} +deb http://{{ sane_debian_system_mirror }}/debian {{ sane_debian_system_codename }} main diff --git a/roles/sshd/README b/roles/sshd/README new file mode 100644 index 0000000..4b155c8 --- /dev/null +++ b/roles/sshd/README @@ -0,0 +1,23 @@ +This role, sshd, configures an SSH server on a Debian. Specifically +may: + +- set host key and certificate +- set user CA +- set port on which server listens + +To use, define variables below: + +- `sshd_version`---must match the current version for the role +- `sshd_host_key` and `sshd_host_cert`---the host key and + corresponding certificate + - note that you must define both for either to work + - rationale: there's little point in just setting the host key, as + it will still force people to accept it the first time; a host + certificate removes that need and allows the key to change at will +- `sshd_port`---the port where the SSH server should listen + - rationale: on public-facing servers, the default port gets tons of + login attempts by attackers trying to guess passwords +- `sshd_user_ca_pub`---the public keys of the SSH CAs trusted to + certify users + - rationale: using a user CA removes the need to maintain, or have, + `authorized_keys` files diff --git a/roles/sshd/defaults/main.yml b/roles/sshd/defaults/main.yml new file mode 100644 index 0000000..20c9563 --- /dev/null +++ b/roles/sshd/defaults/main.yml @@ -0,0 +1,9 @@ +# The user of the role MUST define the version they want to use. If +# it's not what the version of unix_users being used actually +# provides, the role will fail. +sshd_version: null + + +# Allow SSH server to use `authorized_keys` files? +sshd_allow_authorized_keys: yes + diff --git a/roles/sshd/handlers/main.yml b/roles/sshd/handlers/main.yml new file mode 100644 index 0000000..c4898c0 --- /dev/null +++ b/roles/sshd/handlers/main.yml @@ -0,0 +1,4 @@ +- name: sshd_restart + systemd: + name: ssh + state: restarted diff --git a/roles/sshd/subplot.md b/roles/sshd/subplot.md new file mode 100644 index 0000000..e86e513 --- /dev/null +++ b/roles/sshd/subplot.md @@ -0,0 +1,27 @@ +# Role `sshd` – configure an SSH server + +This role sets up a Debian system so that it can be managed with +Ansible in a reasonable way. + +## Version history + +### Version 1 + +First version. Supports `sshd_version`, `sshd_port`, `sshd_host_key`, +`sshd_host_cert`, and `sshd_user_ca_pub`. + +# Configure SSH + +~~~scenario +given a host running Debian +when I use role sshd +and I use variables from sshd.yml +and I run the playbook +then stdout contains "sshd role version" +~~~ + +~~~{#sshd.yml .file .yaml} +ansible_python_interpreter: /usr/bin/python3 + +sshd_version: 1 +~~~ diff --git a/roles/sshd/tasks/main.yml b/roles/sshd/tasks/main.yml new file mode 100644 index 0000000..ff77c40 --- /dev/null +++ b/roles/sshd/tasks/main.yml @@ -0,0 +1,112 @@ +- name: "sshd role version" + shell: | + [ "{{ sshd_version }}" = "1" ] || \ + (echo "Unexpected version {{ sshd_version }}" 1>&2; exit 1) + +- name: "sshd role configuration sanity check" + when: not sshd_allow_authorized_keys and sshd_user_ca_pub is not defined + shell: | + echo "You MUST define sshd_allow_authorized_keys OR sshd_user_ca_pub" + exit 1 + +- name: "Configure SSH server to read config files in sshd_config.d" + lineinfile: + path: /etc/ssh/sshd_config + regexp: "Include /etc/ssh/sshd_config.d" + line: "Include /etc/ssh/sshd_config.d/*.conf" + insertbefore: BOF + notify: sshd_restart + +- name: "Set SSH host identity" + when: sshd_host_key is defined and sshd_host_cert is defined + copy: + content: | + {{ sshd_host_key }} + dest: /etc/ssh/ssh_host_key + owner: root + group: root + mode: 0600 + notify: sshd_restart + +- name: "Set SSH host certificate" + when: sshd_host_key is defined and sshd_host_cert is defined + copy: + content: | + {{ sshd_host_cert }} + dest: /etc/ssh/ssh_host_key-cert.pub + notify: sshd_restart + +- name: "Configure SSH server host key" + when: sshd_host_key is defined and sshd_host_cert is defined + copy: + content: | + HostKeyAlgorithms ssh-ed25519,ssh-ed25519-cert-v01@openssh.com + HostKey /etc/ssh/ssh_host_key + HostCertificate /etc/ssh/ssh_host_key-cert.pub + dest: /etc/ssh/sshd_config.d/host_id.conf + notify: sshd_restart + +- name: "Remove old host key settings from /etc/ssh/sshd_config" + when: sshd_host_key is defined and sshd_host_cert is defined + lineinfile: + path: /etc/ssh/sshd_config + state: absent + regex: "(?i)hostkey" + notify: sshd_restart + +- name: "Remove old host cert settings from /etc/ssh/sshd_config" + when: sshd_host_key is defined and sshd_host_cert is defined + lineinfile: + path: /etc/ssh/sshd_config + state: absent + regex: "(?i)hostcertificate" + notify: sshd_restart + +- name: "Remove old user CA settings from /etc/ssh/sshd_config" + when: sshd_host_key is defined and sshd_host_cert is defined + lineinfile: + path: /etc/ssh/sshd_config + state: absent + regex: "(?i)trustedusercakeys" + notify: sshd_restart + +- name: "Remove obsolete SSH host keys and certificates" + when: sshd_host_key is defined and sshd_host_cert is defined + shell: | + find /etc/ssh -maxdepth 1 -type f -name "ssh_host_*_key*" -delete + notify: sshd_restart + +- name: "Configure SSH server port" + when: sshd_port is defined + copy: + content: | + Port {{ sshd_port }} + dest: /etc/ssh/sshd_config.d/port.conf + notify: sshd_restart + +- name: "Configure user CA for SSH server" + when: sshd_user_ca_pub is defined + copy: + content: | + {{ sshd_user_ca_pub }} + dest: /etc/ssh/user_ca_pubs + notify: sshd_restart + +- name: "Configure SSH server to accept user CA" + when: sshd_user_ca_pub is defined + copy: + content: | + TrustedUserCAKeys /etc/ssh/user_ca_pubs + dest: /etc/ssh/sshd_config.d/user_ca.conf + notify: sshd_restart + +- name: "Configure SSH server to not use 'authorized_keys' files at all." + when: not sshd_allow_authorized_keys + copy: + content: | + AuthorizedKeysFile none + dest: /etc/ssh/sshd_config.d/authorized_keys.conf + notify: sshd_restart + +- name: "Run handlers" + meta: flush_handlers diff --git a/roles/unix_users/defaults/main.yml b/roles/unix_users/defaults/main.yml index ea33fc9..cfcf754 100644 --- a/roles/unix_users/defaults/main.yml +++ b/roles/unix_users/defaults/main.yml @@ -1,3 +1,9 @@ +# The user of the role MUST define the version they want to use. If +# it's not what the version of unix_users being used actually +# provides, the role will fail. +unix_users_version: null + + # List of system users to create. Value a list of dicts with keys: # # username -- the username of the new user @@ -7,7 +13,6 @@ # sudo -- yes/no, should user have sudo access? (without password) # ssh_key -- install this as ~/.ssh/id_rsa # ssh_key_pub -- install this as ~/.ssh/id_rsa.pub -# ssh_key_pub -- install this as ~/.ssh/id_rsa.pub # authorized_keys -- install this as ~/.ssh/authorized_keys # password -- encrypted password # @@ -16,19 +21,3 @@ # unix_users: [] - - -# Specify directory where per-user authorized_keys files are stored. -# Each user has their own file in the directory, named after their -# username. You MUST specify this variable. You may put more than one -# key in each user's file. -# -# You MUST create a file for each user in unix_users. An empty file -# will do. -# -# THIS IS NOW DEPRECATED. DO NOT USE. If you leave this empty, the old, -# deprecated way of installing authorized_keys files is skipped. If you -# still use that, then set it in your own vars. But switch to the new -# way asap: set authorized_keys field for the user, see above. - -authkeys_dir: diff --git a/roles/unix_users/subplot.md b/roles/unix_users/subplot.md new file mode 100644 index 0000000..c7929e9 --- /dev/null +++ b/roles/unix_users/subplot.md @@ -0,0 +1,62 @@ +# Role `unix_users` – manage Unix users + +This role creates or updates Unix users. + +## Configuration + +This role makes use of the following variables: + +* `unix_users_version` – MANDATORY: The playbook should set this + to the version of the role it expects to use. + +* `unix_users` – OPTIONAL: A list of Unix accounts to create. + Defaults to the empty list. Each item in the list is a dict with the + following keys: + + * `username` – MANDATORY: the username of the account + * `comment` – OPTIONAL: the real name (or GECOS field) of the + new account + * `shell` – OPTIONAL: the login shell + * `system` – OPTIONAL: boolean, is this a system user? + * `sudo` – OPTIONAL: boolean, should the account have password-less sudo? + * `ssh_key` – OPTIONAL: text of key to install as `~/.ssh/id_rsa` + * `ssh_key_pub` – OPTIONAL: text of key to install as `~/.ssh/id_rsa.pub` + * `authorized_keys` – OPTIONAL: text of contents of + `~/.ssh/authorized_keys` + * `password` – OPTIONAL: encrypted password + * `groups` – OPTIONAL: list of additional groups to which user + should be added + +Create the encrypted password with something like: + +~~~yaml +password: "{{ lookup('pipe', 'pass show foo | mkpasswd --method=sha-512 --stdin') }}" +~~~ + +## Create normal user with unix_users + +~~~scenario +given a host running Debian +then the host has no user foo +when I use role unix_users +and I use variables from foo.yml +and I run the playbook +then the host has user foo +and the user foo on host has encrypted password foopass +and the user foo on host has shell /bin/true +and the user foo on host has authorized_keys containing "ssh-rsa" +and the user foo on host is in group operator +~~~ + +~~~{#foo.yml .file .yaml} +unix_users_version: 2 + +unix_users: +- username: foo + comment: Foo Bar + shell: /bin/true + password: foopass + authorized_keys: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKVaQfxzzwpwk763IcPBs308TpYYp6+NTOMvYaj3j3ewz8feYQg3lOlKo/5xaPug2ZywG6v6tpn/p0drovT5YAIPJitP7yJAfEzJe/gO7c9uwx0uIpe6cc8bwRG0XFdUVK0EneB6LpIec+3juj4zitGBm0ffIoLDhJ7J0daTzQN62rZaw/2SjSvgbfnu3a2BYRPz1NGiXdvOCbytVSLlUAR6SxNPrFdh/BJnS4umyDaBL/1j2yaw/WlkfZPn5Ni3USZLRcbHnBUUbo64iwBwJabhdpeh0xLGTqDkaeudUgZjlrRHFyCbwJTPtDzJsPLb5HKGGzdXPHP7Lk6PM2CIOz liw@exolobe1 + groups: [operator] +~~~ diff --git a/roles/unix_users/subplot.py b/roles/unix_users/subplot.py new file mode 100644 index 0000000..05330fd --- /dev/null +++ b/roles/unix_users/subplot.py @@ -0,0 +1,59 @@ +import logging + + +def host_does_not_have_user(ctx, username=None): + assert_ne = globals()["assert_ne"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["getent", "passwd", username]) + assert_ne(exit, 0) + + +def host_has_user(ctx, username=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["getent", "passwd", username]) + assert_eq(exit, 0) + output = output.decode("UTF8") + assert f"{username}:" in output + + +def host_user_has_shell(ctx, username=None, shell=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["getent", "passwd", username]) + assert_eq(exit, 0) + for line in output.decode("UTF8").splitlines(): + if line.startswith(f"{username}:"): + logging.debug(f"host_user_has_shell: line={line!r}") + assert line.endswith(f":{shell}") + + +def host_user_has_password(ctx, username=None, password=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["sudo", "grep", f"^{username}:", "/etc/shadow"]) + assert_eq(exit, 0) + for line in output.decode("UTF8").splitlines(): + if line.startswith(f"{username}:"): + parts = line.split(":") + assert_eq(parts[1], password) + + +def host_user_has_authorized_keys_containing(ctx, username=None, substring=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["sudo", "cat", f"/home/{username}/.ssh/authorized_keys"]) + assert_eq(exit, 0) + output = output.decode("UTF8") + assert substring in output + + +def host_user_is_in_group(ctx, username=None, group=None): + assert_eq = globals()["assert_eq"] + qemu = ctx["qemu"] + output, exit = qemu.ssh(["sudo", "-u", username, "groups"]) + assert_eq(exit, 0) + output = output.decode("UTF8") + groups = output.split() + logging.debug(f"host_user_is_in_group: groups={groups}") + assert group in groups diff --git a/roles/unix_users/subplot.yaml b/roles/unix_users/subplot.yaml new file mode 100644 index 0000000..e495602 --- /dev/null +++ b/roles/unix_users/subplot.yaml @@ -0,0 +1,29 @@ +- then: the host has no user {username} + impl: + python: + function: host_does_not_have_user + +- then: the host has user {username} + impl: + python: + function: host_has_user + +- then: the user {username} on host has encrypted password {password} + impl: + python: + function: host_user_has_password + +- then: the user {username} on host has shell {shell} + impl: + python: + function: host_user_has_shell + +- then: the user {username} on host has authorized_keys containing "{substring}" + impl: + python: + function: host_user_has_authorized_keys_containing + +- then: the user {username} on host is in group {group} + impl: + python: + function: host_user_is_in_group diff --git a/roles/unix_users/tasks/main.yml b/roles/unix_users/tasks/main.yml index 00c49fd..e181054 100644 --- a/roles/unix_users/tasks/main.yml +++ b/roles/unix_users/tasks/main.yml @@ -1,3 +1,8 @@ +- name: "check unix_users_version" + shell: | + [ "{{ unix_users_version }}" = "2" ] || \ + (echo "Unexpected version {{ unix_users_version }}" 1>&2; exit 1) + - name: create system users with_items: "{{ unix_users }}" user: @@ -5,6 +10,13 @@ comment: "{{ item.comment|default('unnamed user') }}" shell: "{{ item.shell|default('/bin/bash') }}" system: "{{ item.system|default('no') }}" + +- name: add users to additional groups + with_items: "{{ unix_users }}" + when: item.groups is defined + user: + name: "{{ item.username }}" + groups: "{{ item.groups }}" - name: set password for users with_items: "{{ unix_users }}" @@ -43,14 +55,7 @@ group: "{{ item.username }}" mode: 0600 -- name: add keys to authorized_keys (deprecated way) - with_items: "{{ unix_users }}" - when: authkeys_dir != None - authorized_key: - user: "{{ item.username }}" - key: "{{ lookup('file', authkeys_dir + '/' + item.username) }}" - -- name: add keys to authorized_keys (new way) +- name: add keys to authorized_keys with_items: "{{ unix_users }}" when: item.authorized_keys is defined authorized_key: diff --git a/ssh.config.in b/ssh.config.in new file mode 100644 index 0000000..3f1e3f6 --- /dev/null +++ b/ssh.config.in @@ -0,0 +1,10 @@ +host qemu +hostname localhost +user debian +port <PORT> +userknownhostsfile /dev/null +passwordauthentication no +stricthostkeychecking accept-new +identityfile ssh/test.key +controlmaster auto +controlpersist 60s @@ -0,0 +1,27 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAQEAylWkH8c88KcJO+tyHDwbN9PE6WGKevjUzjL2Go9493sM/H3mEIN5 +TpSqP+cWj7oNmcsBur+raZ/6dHa6L0+WACDyYrT+8iQHxMyXv4Du3PbsMdLiKXunHPG8ER +tFxXVFStBJ3gei6SHnPt47o+M4rRgZtH3yKCw4SeydHWk80Detq2WsP9ko0r4G357t2tgW +ET89TRol3bzgm8rVUi5VAEeksTT6xXYfwSZ0uLpsg2gS/9Y9smsP1pZH2T5+TYt1EmS0XG +x5wVFG6OuIsAcCWm4XaXodMSxk6g5GnrnVIGY5a0Rxcgm8CUz7Q8ybDy2+Ryhhs3Vzxz+y +5OjzNgiDswAAA8i+UR6evlEengAAAAdzc2gtcnNhAAABAQDKVaQfxzzwpwk763IcPBs308 +TpYYp6+NTOMvYaj3j3ewz8feYQg3lOlKo/5xaPug2ZywG6v6tpn/p0drovT5YAIPJitP7y +JAfEzJe/gO7c9uwx0uIpe6cc8bwRG0XFdUVK0EneB6LpIec+3juj4zitGBm0ffIoLDhJ7J +0daTzQN62rZaw/2SjSvgbfnu3a2BYRPz1NGiXdvOCbytVSLlUAR6SxNPrFdh/BJnS4umyD +aBL/1j2yaw/WlkfZPn5Ni3USZLRcbHnBUUbo64iwBwJabhdpeh0xLGTqDkaeudUgZjlrRH +FyCbwJTPtDzJsPLb5HKGGzdXPHP7Lk6PM2CIOzAAAAAwEAAQAAAQEAh8izbPQbTHD8fG7E +VHht16hRdEGWWnJU9dAzYp24E3VLwMKIu7pPlVGlc18Uv/2fFP+suHPah/bpcHEg/5EMXC +fAIkfO9BcD86lNiSHwqu82kTUxu58VBhKgIGbKCvppNwzTFaLQTF4JPyKKqbBaH6eV0I/Z +C+apG8sjoVI3ko8oKjjwTQ/oHHC71APXmLznhupxF2ohHf0jZW3g2Ktc86AiL0JUwmE6nb +FrZxCTBaYSaa7pQSlgJOjR0+xqLcf7a/rechXlAknKnFCBU2j2rlBMFy8QZUS0ADY1sWcq +jMRksBVSdPgfC5Ki/a8JWERhX6QoSnvnNR2RJxWpcPs1eQAAAIACOBS8WM26pw5OhDorV2 +NJIqwkzT8fMjjCuTACsa75wtqO4c61rOYy9K0C0/ukTmGHwi9HhjLTC9Ndhi9cWMKOyQL+ +UMKaurmnvtu39odLAG3CGE/SIM6Z6WyyGVpy/FJweTbDKAP7ldr1J49W+dIaCSPyHyatYT +Fwa7OsJRga5gAAAIEA+mvBnY9U9dj6NVZZAWHF4umSI+VuVmZl8NI0DMTG2hIAPbqi2Eez +zxp3195mc2Tf9YM81YqC1MVn3nqYJ5Kx+kibbTBphAWcbQe6dhn6zR8jKc8FNA8kdTJi4i +lvQ2wTEmrJtv2dEMkSNEZOrVw0eAZ9AuDBtcCvmzeRTAt0eQUAAACBAM7XoVeGHIaVA/GL +skuyNb7J56rCnMXAMJ9jgoaZxx3ON9DDfz8YiAnL1drzdkAtUNL0QlvVQuFlzAUlCPzCHO +HSEUEW7VFoejy+WSxoP6+F1qVQNnr38OQEswkZSko/+yiDMdgiAAUPaE8v0UZerJg8quHB +gDal1friWML9x0dXAAAADGxpd0BleG9sb2JlMQECAwQFBg== +-----END OPENSSH PRIVATE KEY----- diff --git a/ssh/id.pub b/ssh/id.pub new file mode 100644 index 0000000..714c5cc --- /dev/null +++ b/ssh/id.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKVaQfxzzwpwk763IcPBs308TpYYp6+NTOMvYaj3j3ewz8feYQg3lOlKo/5xaPug2ZywG6v6tpn/p0drovT5YAIPJitP7yJAfEzJe/gO7c9uwx0uIpe6cc8bwRG0XFdUVK0EneB6LpIec+3juj4zitGBm0ffIoLDhJ7J0daTzQN62rZaw/2SjSvgbfnu3a2BYRPz1NGiXdvOCbytVSLlUAR6SxNPrFdh/BJnS4umyDaBL/1j2yaw/WlkfZPn5Ni3USZLRcbHnBUUbo64iwBwJabhdpeh0xLGTqDkaeudUgZjlrRHFyCbwJTPtDzJsPLb5HKGGzdXPHP7Lk6PM2CIOz liw@exolobe1 diff --git a/subplot.md b/subplot.md new file mode 100644 index 0000000..a56e15b --- /dev/null +++ b/subplot.md @@ -0,0 +1,64 @@ +# Introduction + +`debian-ansible` is a collection of Ansible roles for managing Debian +systems. The roles are re-usable and parameterised so that they can be +adapted to some variations. + +This document describes the roles and also acts as an acceptance test +suite for them. The [Subplot][] program can generate a test program +based on this document, and the test program tests that the roles work +as intended. + +[Subplot]: https://subplot.liw.fi/ + +## Using these roles + +Eventually, all the roles included with `debian-ansible` will follow +the same principles: + +* the playbook using a role defines which version of each role they + expect to use by defining a variable such as `unix_users_version` + - the role checks that it's defined to the right value and fails if + it isn't +* the role is parameterized via variables, with names prefixed with + the role name + - the role `unix_users` expects a variable `unix_users` that lists + all the users to create +* any variables used by role have "empty" default values, unless the + variable must be defined by the playbook using the role + - the role will check that the necessary variables are defined, + before doing anything + +# Implementation and testing of these roles + +If you're just using the roles, you don't need to care about this chapter. + +## Source files + +At the root of the `debian-ansible` source tree is a `subplot.md`, and +the subdirectory `subplot` with some other files used by the main +file. Each role directory should have a `subplot.md`, and may also +have `subplot.yaml` and `subplot.py`. The files in role directories +get combined with the main ones the the `./check` script. + +## Scenario structure + +Each `subplot.md` file is meant to contain at least one scenario. All +scenarios have the same structure: + +* create a new VM +* construct an Ansible playbook that uses the role in a specific way +* run the playbook against the VM +* examine the VM to verify it has been changed in the expected way + +The VM is created using a base image, which the user must specify to +the test program by setting the `BASEIMAGE` environment variable. + +## Sanity check + +Verify that everything looks OK and that other scenarios can be run. + +~~~scenario +given a host running Debian +then I can run /bin/true on the host +~~~ diff --git a/subplot.subplot b/subplot.subplot new file mode 100644 index 0000000..5d8083f --- /dev/null +++ b/subplot.subplot @@ -0,0 +1,11 @@ +title: "debian-ansible—Ansible roles for Debian systems" +authors: + - Lars Wirzenius +markdowns: + - subplot.md +bindings: +- subplot.yaml +impls: + python: + - subplot.py + - lib/runcmd.py diff --git a/subplot/qemumgr.py b/subplot/qemumgr.py new file mode 100755 index 0000000..c4fd385 --- /dev/null +++ b/subplot/qemumgr.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 + + +import logging +import os +import random +import shlex +import shutil +import signal +import subprocess +import tempfile +import time + + +CLOUD_INIT_META_TEMPLATE = """\ +# Amazon EC2 style metadata +local-hostname: {hostname} +""" + +CLOUD_INIT_USER_TEMPLATE = """\ +#cloud-config +ssh_authorized_keys: +- {pubkey} +""" + + +class QemuSystem: + def __init__(self, name, base_image, disk_size, pubkey): + self._name = name + self._dirname = "." + self._image = self._copy_image(base_image, disk_size) + self._pubkey = pubkey + self._memory = 512 + self._vcpus = 1 + self._username = None + self._port = None + self._pid = None + + fd, self._knownhosts = tempfile.mkstemp() + os.close(fd) + + def _join(self, *names): + return os.path.join(self._dirname, *names) + + def set_disk_size(self, size): + self._disk_size = str(size * 1024 * 1024) + + def set_memory(self, memory): + self._memory = memory + + def set_vcpus(self, vcpus): + self._vcpus = vcpus + + def set_username(self, username): + self._username = username + + def get_port(self): + return self._port + + def _copy_image(self, base_image, size): + image = self._join("qemu.img") + logging.debug(f"QemuSystem: actual disk image: {image}") + shutil.copy(base_image, image) + if size is not None: + subprocess.check_call(["qemu-img", "resize", "-q", image, str(size)]) + return image + + def _cloud_init_iso(self): + iso = self._join("cloud-init.iso") + + config = self._join("cloud-init") + os.mkdir(config) + + meta = os.path.join(config, "meta-data") + write_file(meta, CLOUD_INIT_META_TEMPLATE.format(hostname=self._name)) + + user = os.path.join(config, "user-data") + write_file(user, CLOUD_INIT_USER_TEMPLATE.format(pubkey=self._pubkey)) + + if os.path.exists(iso): + os.remove(iso) + subprocess.check_call( + [ + "genisoimage", + "-quiet", + "-joliet", + "-rock", + "-output", + iso, + "-volid", + "cidata", + config, + ] + ) + + return iso + + def start(self): + iso = self._cloud_init_iso() + + self._port = random.randint(2000, 30000) + pid_file = self._join("qemu.pid") + out_file = self._join("qemu.out") + err_file = self._join("qemu.err") + + MiB = 1024 ** 2 + memory = int(self._memory / MiB) + + argv = [ + "/usr/bin/daemonize", + "-c", + self._dirname, + "-p", + pid_file, + "-e", + err_file, + "-o", + out_file, + "/usr/bin/qemu-system-x86_64", + "-name", + self._name, + "-m", + str(memory), + "-cpu", + "host", + "-smp", + f"cpus={self._vcpus}", + "-drive", + f"file={self._image},format=qcow2,cache=none,if=virtio", + "-drive", + f"file={iso},if=ide,media=cdrom", + "-device", + "virtio-net,netdev=user.0", + "-netdev", + f"user,id=user.0,hostfwd=tcp::{self._port}-:22", + "-nographic", + "-enable-kvm", + ] + + subprocess.check_call(argv, stdout=None, stderr=None) + self._pid = int(wait_for(lambda: got_pid(pid_file), 1)) + + logging.debug("started qemu-system") + logging.debug(f" argv: {argv}") + logging.debug(f" pid: {self._pid}") + + def stop(self): + if self._pid is not None: + logging.debug(f"killing qemu-system process {self._pid}") + os.kill(self._pid, signal.SIGTERM) + + def wait_for_ssh(self): + def ssh_ok(): + output, exit = self.ssh(["true"]) + logging.debug(f"tried ssh, exit={exit}") + if exit == 0: + return True + return None + + return wait_for(ssh_ok, 300, sleep=5) + + def ssh(self, argv): + assert self._username is not None + + srcdir = globals()["srcdir"] + + ssh_opts = [ + "-i", + os.path.join(srcdir, "ssh", "id"), + "-p", + str(self._port), + "-l", + self._username, + f"-ouserknownhostsfile={self._knownhosts}", + "-ostricthostkeychecking=accept-new", + "-opasswordauthentication=no", + ] + + real_argv = ( + ["ssh"] + ssh_opts + ["localhost", "--"] + [shlex.quote(x) for x in argv] + ) + + logging.debug(f"running on VM: {real_argv}") + p = subprocess.Popen( + real_argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + output, _ = p.communicate() + logging.debug(f"exit code: {p.returncode}") + logging.debug(f"output: {output}") + return output, p.returncode + + +def read_file(filename): + with open(filename, "r") as f: + return f.read() + + +def write_file(filename, content): + with open(filename, "w") as f: + f.write(content) + + +def wait_for(func, timeout, sleep=0.1): + start = time.time() + while time.time() < start + timeout: + val = func() + if val is not None: + return val + time.sleep(sleep) + + +def got_pid(pid_file): + if os.path.exists(pid_file): + pid = read_file(pid_file) + if pid: + return pid.strip() diff --git a/subplot/subplot.py b/subplot/subplot.py new file mode 100644 index 0000000..08472bd --- /dev/null +++ b/subplot/subplot.py @@ -0,0 +1,145 @@ +import logging +import os +import yaml + + +def fixme(ctx, **kwargs): + assert 0 + + +def create_vm(ctx): + QemuSystem = globals()["QemuSystem"] + srcdir = globals()["srcdir"] + + MiB = 1024 ** 2 + GiB = 1024 ** 3 + + cfg = yaml.safe_load(open(os.path.join(srcdir, "test.cfg"))) + name = cfg["name"] + base_image = cfg["base_image"] + username = cfg["username"] + cpus = cfg["cpus"] + memory = cfg["memory"] * MiB + + # We use a hard-coded test key that we have in the source tree. + pubkey = open(os.path.join(srcdir, "ssh", "id.pub")).read().strip() + + # We hard code the disk image size, since we don't expect scenarios to have + # any specific size needs, and also, the qcow2 image format is only as big + # as the data put into the disk, so we can choose a size that fits all. + disk_size = 10 * GiB + + logging.info("starting a VM using qemu-system") + logging.info(f" name : {name}") + logging.info(f" image : {base_image}") + logging.info(f" disk : {disk_size}") + logging.info(f" pubkey : {pubkey}") + logging.info(f" memory : {memory}") + logging.info(f" cpus : {cpus}") + logging.info(f" username: {username}") + + qemu = QemuSystem(name, base_image, disk_size, pubkey) + qemu.set_memory(memory) + qemu.set_vcpus(cpus) + qemu.set_username(username) + qemu.start() + + logging.debug("waiting for SSH to be ready") + if qemu.wait_for_ssh(): + logging.debug("SSH is ready") + else: + logging.error("SSH did not get ready") + assert 0 + logging.info("a qemu-system VM is up and running and accessible over SSH") + ctx["qemu"] = qemu + + +def destroy_vm(ctx): + logging.debug(f"destroying qemu running") + qemu = ctx["qemu"] + qemu.stop() + + +def run_true_on_host(ctx): + qemu = ctx["qemu"] + qemu.ssh(["/bin/true"]) + + +def use_role_in_playbook(ctx, role=None): + empty_playbook = { + "hosts": "test-host", + "remote_user": "debian", # FIXME: don't hardcode this + "become": True, + "roles": [], + } + playbook = ctx.get("playbook", dict(empty_playbook)) + playbook["roles"].append(role) + ctx["playbook"] = playbook + + +def set_vars_file(ctx, filename=None): + get_file = globals()["get_file"] + data = get_file(filename) + with open("vars.yml", "wb") as f: + f.write(data) + + +def try_playbook(ctx): + runcmd_run = globals()["runcmd_run"] + assert_ne = globals()["assert_ne"] + srcdir = globals()["srcdir"] + + qemu = ctx["qemu"] + + with open("hosts", "w") as f: + f.write("test-host\n") + + if not os.path.exists("vars.yml"): + with open("vars.yml", "w") as f: + yaml.safe_dump({}, stream=f) + + playbook = [ctx["playbook"]] + assert_ne(playbook, None) + with open("playbook.yml", "w") as f: + yaml.safe_dump(playbook, stream=f) + + ssh_opts = [ + "-ouserknownhostsfile=/dev/null", + "-ostricthostkeychecking=accept-new", + "-i", + os.path.join(srcdir, "ssh", "id"), + f"-p{qemu.get_port()}", + ] + + env = dict(os.environ) + env["ANSIBLE_SSH_ARGS"] = " ".join(ssh_opts) + env["ANSIBLE_LOG"] = "ansible.log" + env["ANSIBLE_ROLES_PATH"] = os.path.join(srcdir, "roles") + + argv = [ + "ansible-playbook", + "-i", + "hosts", + f"-e@vars.yml", + "-eansible_ssh_host=localhost", + f"-eansible_ssh_port={qemu.get_port()}", + "playbook.yml", + ] + + runcmd_run(ctx, argv, env=env) + + +def command_fails(ctx): + runcmd_exit_code_is_nonzero = globals()["runcmd_exit_code_is_nonzero"] + runcmd_exit_code_is_nonzero(ctx) + + +def run_playbook(ctx): + runcmd_exit_code_is_zero = globals()["runcmd_exit_code_is_zero"] + try_playbook(ctx) + runcmd_exit_code_is_zero(ctx) + + +def xstdout_contains(ctx, text=None): + runcmd_stdout_contains = globals()["runcmd_stdout_contains"] + runcmd_stdout_contains(ctx, text=text) diff --git a/subplot/subplot.yaml b/subplot/subplot.yaml new file mode 100644 index 0000000..286e878 --- /dev/null +++ b/subplot/subplot.yaml @@ -0,0 +1,42 @@ +- given: a host running Debian + impl: + python: + function: create_vm + cleanup: destroy_vm + +- then: I can run /bin/true on the host + impl: + python: + function: run_true_on_host + +- when: I use role {role} + impl: + python: + function: use_role_in_playbook + +- when: I use variables from {filename} + impl: + python: + function: set_vars_file + types: + filename: file + +- when: I run the playbook + impl: + python: + function: run_playbook + +- when: I try to run the playbook + impl: + python: + function: try_playbook + +- then: the command fails + impl: + python: + function: command_fails + +- then: stdout contains "{text:text}" + impl: + python: + function: xstdout_contains |