feat: replace options with a YAML specification file

Files can be kept in git. Command line invocation are less convenient.

Also: discard all the drives, allow extra LVs to be created, and
refactor the code to be easier to follow.

Sponsored-by: author

3 files changed, 316 insertions, 227 deletions

diff --git a/installer.vmdb b/installer.vmdb
index 218a809..9379f1b 100644
--- a/installer.vmdb
+++ b/installer.vmdb
@@ -15,7 +15,7 @@ steps:
     device: "{{ output }}"
     start: 1G
     end: 100%
-    tag: /
+    tag: root
 
   - kpartx: "{{ output }}"
 
@@ -23,24 +23,24 @@ steps:
     partition: efi
 
   - mkfs: ext4
-    partition: /
+    partition: root
 
-  - mount: /
+  - mount: root
 
-  - unpack-rootfs: /
+  - unpack-rootfs: root
 
   - debootstrap: bullseye
     mirror: http://deb.debian.org/debian
-    target: /
+    target: root
     unless: rootfs_unpacked
 
   - apt: install
     packages:
       - linux-image-amd64
-    fs-tag: /
+    fs-tag: root
     unless: rootfs_unpacked
 
-  - cache-rootfs: /
+  - cache-rootfs: root
     unless: rootfs_unpacked
 
   - apt: install
@@ -55,18 +55,23 @@ steps:
       - lvm2
       - cryptsetup
       - cryptsetup-initramfs
-#      - pass
       - dosfstools
-#      - emacs
-#      - gpg
-#      - scdaemon
-    tag: /
+    tag: root
 
-  - ansible: /
-    playbook: v-i.yml
+  - ansible: root
+    playbook: installer.yml
 
-  - fstab: /
+  - fstab: root
+
+  - copy-file: /root/vi
+    src: v-i
+    perm: 0755
+
+  - copy-file: /root/std.yml
+    src: std.yml
+
+  - zerofree: root
 
   - grub: uefi
-    tag: /
+    tag: root
     efi: efi
diff --git a/std.yml b/std.yml
index 75aca0d..cb62c82 100644
--- a/std.yml
+++ b/std.yml
@@ -1,4 +1,7 @@
 # Ansible playbook to install stuff for a standard install with v-i.
+# You should inspect the user_* variables at the end, and override
+# them with "ansible_vars" in the system spec file. v-i sets the
+# hostname variable automatically.
 
 - hosts: image
   tasks:
@@ -8,11 +11,6 @@
           {{ hostname }}
         dest: /etc/hostname
 
-    - name: "remove root password"
-      shell: |
-        # passwd -l root
-        sed -i '/^root:[^:]*:/root::/' /etc/passwd
-
     - name: "create ~root/.ssh"
       file:
         state: directory
@@ -57,19 +55,10 @@
           {{ user_locale }}
         dest: /etc/profile.d/finnish.sh
 
-    - name: "configure Ethernet networking"
-      copy:
-        content: |
-          auto eth0
-          iface eth0 inet dhcp
-          iface eth0 inet6 auto
-        dest: /etc/network/interfaces.d/wired
-
-    # - name: "restrict root logins over ssh"
-    #   lineinfile:
-    #     path: /etc/ssh/sshd_config
-    #     regex: "#* *PasswordAuthentication"
-    #     line: "PasswordAuthentication no"
+    - name: "remove ifupdown"
+      apt:
+        name: ifupdown
+        state: absent
 
     - name: "configure networkd"
       copy:
@@ -81,54 +70,17 @@
           DHCP=yes
         dest: /etc/systemd/network/external.network
 
-    - name: "remove ifupdown"
-      apt:
-        name: ifupdown
-        state: absent
-
     - name: "enable networkd"
       systemd:
         name: systemd-networkd
         enabled: yes
 
   vars:
-    hostname: v-i
-    user_pub: |
-      ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPQe6lsTapAxiwhhEeE/ixuK+5N8esCsMWoekQqjtxjP liw personal systems
+    ansible_python_interpreter: /usr/bin/python3
+
+    # You may want to override these.
     user_locale: |
       LC_CTYPE=fi_FI.UTF8
     user_keyboard_model: pc105
     user_keyboard_layout: fi
     user_console_codeset: Lat15
-
-    ansible_python_interpreter: /usr/bin/python3
-    ci_prod_signing_key: |
-        -----BEGIN PGP PUBLIC KEY BLOCK-----
-
-        mQINBFrLO7kBEADdz6mHstYmKU5Dp6OSjxWtWaqTDOX1sJdmmaIK/9EKVIH0Maxp
-        5kvVO5G6mULLAjv/kLG0MxasHPrq8I2A/y8AqKAGVL8QelwLjQMIFZ30/VbGQPHS
-        +T5TZXEnoQtNce1GUhFwJ38ZyjjwHBFV9tSec7rZ2Q3YeM3nNnGPf6DacXGfEOPO
-        HIN4sXAN2hzNXNjKRzTIvxQseb6nr7afUh/SlZ3yhQOCrIzmYlD7tP9WJe7ofL0p
-        JY4pDQYw8rT6nC2BE/ioemh84kERCT1vCe+OVFlSRuMlqfEv+ZpKQ+itOmPDQ/lM
-        jpUm1K2hrW/lWpxT/ZxHKo/w1K36J5WshgMZxfUu5BMCL9LMqMcrXNhNjDMfxDMM
-        3yBPOvQ4ls6fecOZ/bsFo1p8VzMk/w/eG8vPs5yuNa5XxN95yFMXoOHGb5Xbu8D4
-        6yiW+Af70LbiSNpGdmNdneiGB2fY38NxBukPw5u3S5qG8HedSmMr1RvSr5kHoAAe
-        UbOY+BYaaKsTAT7+1skUW1o3FJSqoRKCHAzTsMWC6zzhR8hRn7jVrrguH1hGbqq5
-        TZSCFQZExuTJ7uXrTLG0WoBXIjB5wWNcSeXn8myUWYB51nJNF4tJBouZOz9JwWGl
-        kiAQkrHnBttLQWdW9FyjbIoTZMtpvVx+m6ObGTGdGL1cNlLAvWprMXGc+QARAQAB
-        tDJJY2sgQVBUIHJlcG9zaXRvcnkgc2lnbmluZyBrZXkgKDIwMTgpIDxsaXdAbGl3
-        LmZpPokCTgQTAQgAOBYhBKL1uyDoXyxUH3O717Wr+TZVS6PGBQJayzu5AhsDBQsJ
-        CAcCBhUICQoLAgQWAgMBAh4BAheAAAoJELWr+TZVS6PGB5QQANTcikhRUHwt9N4h
-        dGc/Hp6CbqdshMoWlwpFskttoVDxQG5OAobuZl5XyzGcmja1lT85RGkZFfbca0IZ
-        LnXOLLSAu51QBkXNaj4OhjK/0uQ+ITrvL6RQSXNgHiUTR/W2XD1GIUq6nBqe2GSN
-        31S1baYKKVj5QIMsi7Dq8ls3BBXuPCE+xTSaNmGWjes2t9pPidcRvxsksCLY1qgw
-        P1GFXBeMkBQ29kBP87SUL15SIk7OiQLlEURCy5iRls5rt/YEsdEpRWIb0Tm5Nrjv
-        2M3VM+iBhfNXTwj0rJ34mlycF1qQmA7YcTEobT7z587GPY0VWzBpQUnEQj7rQWPM
-        cDYY0b+I6kQ8VKOaL4wVAtE98d7HzFIrIrwhTKufnrWrVDPYsmLZ+LPC1jiF7JBD
-        SR6Vftb+SdDR9xoE1yRuXbC6IfoW+5/qQNrdQ2mm9BFw5jOonBqchs18HTTf3441
-        6SWwP9fY3Vi+IZphPPi0Gf85oMStgnv/Wnw6LacEL32ek39Desero/D8iGLZernK
-        Q2mC9mua5A/bYGVhsNWyURNFkKdbFa+/wW3NfdKYyZnsSfo+jJ2luNewrhAY7Kod
-        GWXTer9RxzTGA3EXFGvNr+BBOOxSj0SfWTl0Olo7J5dnxof+jLAUS1VHpceHGHps
-        GSJSdir7NkZidgwoCPA7BTqsb5LN
-        =dXB0
-        -----END PGP PUBLIC KEY BLOCK-----
diff --git a/v-i b/v-i
index c1957fa..2f613a7 100755
--- a/v-i
+++ b/v-i
@@ -82,8 +82,8 @@ def is_luks(path):
     return p.returncode == 0
 
 
-def clean_up_disks(device):
-    log("clean up disks from old installs")
+def clean_up_disks(drives):
+    log(f"clean system LVM2 and drives from everything: {drives}")
 
     mounts = mount_points()
     pvs = physical_volumes()
@@ -115,118 +115,223 @@ def clean_up_disks(device):
             log(f"open LUKS volume {mapping} (just in case it is one)")
             run(["cryptsetup", "close", mapping], check=False, capture_output=True)
 
-    log(f"blkdiscard {device}")
-    run(["blkdiscard", "--force", device], check=True, capture_output=True)
+    for drive in drives:
+        log(f"blkdiscard {drive}")
+        run(
+            ["blkdiscard", "--force", drive],
+            check=True,
+            capture_output=True,
+        )
 
 
-def vmdb_spec(cryptsetup_password, playbook, playbook2, extra_vars):
-    device = "{{ image }}"
-    spec = {
-        "steps": [
-            {
-                "mklabel": "gpt",
-                "device": device,
-            },
-            {
-                "mkpart": "primary",
-                "device": device,
-                "start": "0%",
-                "end": "500M",
-                "tag": "efi",
-            },
-            {
-                "mkpart": "primary",
-                "device": device,
-                "start": "500M",
-                "end": "1G",
-                "tag": "boot",
-            },
-        ]
+def mklabel(device):
+    return {
+        # Create a GPT partition table. It works well for all modern
+        # systems.
+        "mklabel": "gpt",
+        "device": device,
+    }
+
+
+def mkpart(device, tag, start, end):
+    return {
+        "mkpart": "primary",
+        "device": device,
+        "start": start,
+        "end": end,
+        "tag": tag,
+    }
+
+
+def mkfs(tag, fstype):
+    return {
+        "mkfs": fstype,
+        "partition": tag,
+    }
+
+
+def cryptsetup(tag, password, name):
+    return {
+        "cryptsetup": "cryptsetup0",
+        "password": password,
+        "name": name,
     }
 
+
+def vgcreate(tag, drives):
+    return {
+        "vgcreate": tag,
+        "physical": drives,
+    }
+
+
+def lvcreate(vg, name, size):
+    return {
+        "lvcreate": vg,
+        "name": name,
+        "size": size,
+    }
+
+
+def mount(tag, dirname=None, mount_on=None):
+    d = {"mount": tag}
+    if dirname is not None:
+        d["dirname"] = dirname
+    if mount_on is not None:
+        d["mount-on"] = mount_on
+    return d
+
+
+def unpack_rootfs(tag):
+    return {
+        "unpack-rootfs": tag,
+    }
+
+
+def cache_rootfs(tag):
+    return {
+        "cache-rootfs": tag,
+        "unless": "rootfs_unpacked",
+    }
+
+
+def debootstrap(tag):
+    return {
+        "debootstrap": "bullseye",
+        "mirror": "http://deb.debian.org/debian",
+        "target": tag,
+        "unless": "rootfs_unpacked",
+    }
+
+
+def apt(tag, packages, unless=True):
+    d = {
+        "apt": "install",
+        "packages": [
+            "console-setup",
+            "dosfstools",
+            "linux-image-amd64",
+            "locales-all",
+            "lvm2",
+            "psmisc",
+            "python3",
+            "ssh",
+            "strace",
+        ],
+        "tag": tag,
+    }
+    if unless:
+        d["unless"] = "rootfs_unpacked"
+    return d
+
+
+def virtual_filesystems(tag):
+    return {
+        "virtual-filesystems": "root",
+    }
+
+
+def fstab(tag):
+    return {
+        "fstab": tag,
+    }
+
+
+def grub(device, root, efi):
+    return {
+        "grub": "uefi",
+        "tag": root,
+        "efi": efi,
+        "quiet": True,
+        "image-dev": device,
+    }
+
+
+def vmdb_spec(system, ansible_vars):
+    device = "{{ image }}"
+    steps = [
+        mklabel(device),
+        # Create a partition for UEFI to boot from.
+        mkpart(device, "efi", "0%", "500M"),
+        # Create a separate /boot partition, in case we want to have LUKS.
+        mkpart(device, "boot", "500M", "1G"),
+        # Format the EFI partition. This MUST be vfat.
+        mkfs("efi", "vfat"),
+        # Format /boot. This is conventionally ext2. This file system
+        # gets very little I/O, but it MUST be supported by GRUB, so
+        # ext2 seems like a nice, safe choice.
+        mkfs("boot", "ext2"),
+    ]
+
     # Set up pv0 for lvm2, either encrypted or cleartext.
-    if cryptsetup_password:
-        spec["steps"].extend(
+    if system.luks:
+        steps.extend(
             [
-                {
-                    "mkpart": "primary",
-                    "device": device,
-                    "start": "1G",
-                    "end": "100%",
-                    "tag": "cryptsetup0",
-                },
-                {
-                    "cryptsetup": "cryptsetup0",
-                    "password": cryptsetup_password,
-                    "name": "pv0",
-                },
+                mkpart(device, "cryptsetup0", "1G", "100%"),
+                cryptsetup("cryptsetup0", system.luks, "pv0"),
             ]
         )
+        for (i, drive) in enumerate(system.extra_drives):
+            steps.append(cryptsetup(f"cryptsetuo{i+1}", system.luks, f"pv{i+1}"))
     else:
-        spec["steps"].extend(
+        steps.append(mkpart(device, "pv0", "1G", "100%"))
+        for (i, drive) in enumerate(system.extra_drives):
+            steps.extend(
+                [
+                    mklabel(drive),
+                    mkpart(drive, f"pv{i+1}", "0%", "100%"),
+                ]
+            )
+
+    # Create file systems and install Debian.
+    steps.extend(
+        [
+            # Create an LVM2 volume group using pv0, which we create
+            # earlier. At this point, if LUKS is used, pv0 is the unlocked,
+            # open, cleartext block device.
+            vgcreate(
+                "vg0",
+                [f"pv{i}" for i in range(len([system.drive] + system.extra_drives))],
+            ),
+            # Create a 20 gigabyte LV for the root file system. That's big
+            # enough for a desktop system.
+            lvcreate("vg0", "root", "20G"),
+            # format the root file system. This gets a fair bit of use, so
+            # ext4 seems like a safe choice. If you wanted another file
+            # system, sorry.
+            mkfs("root", "ext4"),
+            # Mount the root file system.
+            mount("root"),
+            # Mount /boot on top of the root file system.
+            mount("boot", dirname="/boot", mount_on="root"),
+            # Mount /boot/efi.
+            mount("efi", dirname="/boot/efi", mount_on="boot"),
+        ]
+    )
+
+    # Add any additional LVs.
+    for lv in system.extra_lvs:
+        steps.extend(
             [
-                {
-                    "mkpart": "primary",
-                    "device": device,
-                    "start": "1G",
-                    "end": "100%",
-                    "tag": "pv0",
-                },
+                lvcreate("vg0", lv["name"], lv["size"]),
+                mkfs(lv["name"], "ext4"),
             ]
         )
 
-    # Create file systems and install Debian.
-    spec["steps"].extend(
+    steps.extend(
         [
-            {
-                "mkfs": "vfat",
-                "partition": "efi",
-            },
-            {
-                "mkfs": "ext2",
-                "partition": "boot",
-            },
-            {
-                "vgcreate": "vg0",
-                "physical": ["pv0"],
-            },
-            {
-                "lvcreate": "vg0",
-                "name": "root",
-                "size": "10G",
-            },
-            {
-                "mkfs": "ext4",
-                "partition": "root",
-            },
-            {
-                "mount": "root",
-            },
-            {
-                "mount": "boot",
-                "dirname": "/boot",
-                "mount-on": "root",
-            },
-            {
-                "mount": "efi",
-                "dirname": "/boot/efi",
-                "mount-on": "boot",
-            },
-            {
-                "unpack-rootfs": "root",
-            },
-            {
-                "debootstrap": "bullseye",
-                "mirror": "http://deb.debian.org/debian",
-                "target": "root",
-                "unless": "rootfs_unpacked",
-            },
-            {
-                "apt": "install",
-                "packages": [
+            # If we have a cached version of the installed system, unpack
+            # it now. Otherwise do nothing. Note that if you make any
+            # changes to the steps marked "unless: rootfs_unpacked", you
+            # have to remember to manually remove the cache file. v-i or
+            # vmdb2 won't do that automatically for you.
+            unpack_rootfs("root"),
+            debootstrap("root"),
+            apt(
+                "root",
+                [
                     "console-setup",
                     "dosfstools",
-                    # "ifupdown",
                     "linux-image-amd64",
                     "locales-all",
                     "lvm2",
@@ -235,84 +340,112 @@ def vmdb_spec(cryptsetup_password, playbook, playbook2, extra_vars):
                     "ssh",
                     "strace",
                 ],
-                "tag": "root",
-                "unless": "rootfs_unpacked",
-            },
-            {
-                "cache-rootfs": "root",
-                "unless": "rootfs_unpacked",
-            },
-            {
-                # This MUST be after the debootstrap step.
-                "virtual-filesystems": "root",
-            },
-            {
-                "fstab": "root",
-            },
-            {
-                # These MUST come after the fstab step so that they add the
-                # crypttab in the initramfs.
-                "apt": "install",
-                "packages": [
+            ),
+            # If we didn't unpack an existing cache archive, make one now.
+            # Otherwise, skip this step.
+            cache_rootfs("root"),
+            # This MUST be after the debootstrap step.
+            virtual_filesystems("root"),
+            # Create /etc/fstab (and, if LUKS is used, /etc/crypttab).
+            fstab("root"),
+            # These MUST come after the fstab step so that they add the
+            # crypttab in the initramfs. We install them regardless of
+            # whether LUKS is used: they're harmless if LUKS isn't used.
+            apt(
+                "root",
+                [
                     "cryptsetup",
                     "cryptsetup-initramfs",
                 ],
-                "tag": "root",
-            },
-            {
-                # This also MUST come outside the rootfs caching, as it install
-                # things outside the file systems.
-                "grub": "uefi",
-                "tag": "root",
-                "efi": "efi",
-                "quiet": True,
-                "image-dev": device,
-            },
+                unless=False,
+            ),
+            # This also MUST come outside the rootfs caching, as it install
+            # things outside the file systems, and those won't be in the
+            # cache.
+            grub(device, "root", "efi"),
         ]
     )
 
     # If playbooks have been specified, add ansible steps.
-    for p in (playbook, playbook2):
+    for p in ["std.yml"] + system.extra_playbooks:
         if p:
-            spec["steps"].append(
-                {"ansible": "root", "playbook": p, "extra_vars": extra_vars}
-            )
+            steps.append({"ansible": "root", "playbook": p, "extra_vars": ansible_vars})
+
+    return {"steps": steps}
+
+
+class SystemSpec:
+    def __init__(self, filename):
+        REQUIRED = "required"
+        self._obj = {
+            "hostname": REQUIRED,
+            "drive": REQUIRED,
+            "extra_drives": [],
+            "extra_lvs": [],
+            "extra_playbooks": [],
+            "ansible_vars": {},
+            "luks": None,
+        }
+        with open(filename) as f:
+            obj = yaml.safe_load(f)
+
+        # Check for unknown keys.
+        for key in obj:
+            if key not in self._obj:
+                sys.exit(f"spec has unknown key: {key}")
+
+        # Check for missing required keys.
+        for key in self._obj:
+            if self._obj[key] == REQUIRED and key not in obj:
+                sys.exit(f"spec lacks required key {key}")
+
+        # Check for types of values.
+        for key in self._obj:
+            if key in obj:
+                e = type(self._obj[key])
+                a = type(obj[key])
+                if a != e:
+                    sys.exit(f"spec key {key} has unexpected type {a}, wanted {e}")
 
-    return spec
+        self._obj.update(obj)
+
+        for key in self._obj:
+            setattr(self, key, self._obj[key])
+        del self._obj
+
+    def __repr__(self):
+        r = {key: getattr(self, key) for key in dir(self) if not key.startswith("_")}
+        return repr(r)
 
 
 def main():
     p = argparse.ArgumentParser()
     p.add_argument("--verbose", action="store_true")
+    p.add_argument("--very-verbose", action="store_true")
     p.add_argument("--log", default="install.log")
     p.add_argument("--cache", default="cache.tar.gz")
-    p.add_argument("--playbook", default="std.yml")
-    p.add_argument("--playbook2")
-    p.add_argument("--vars")
-    p.add_argument("--luks")
-    p.add_argument("device")
+    p.add_argument("spec")
     args = p.parse_args()
 
-    if args.verbose:
-        global verbose
-        verbose = args.verbose
+    global verbose
+    verbose = args.verbose
 
-    extra_vars = {}
-    if args.vars:
-        with open(args.vars) as f:
-            extra_vars = yaml.safe_load(f)
+    system = SystemSpec(args.spec)
+    log(f"spec: {system!r}")
 
-    clean_up_disks(args.device)
+    clean_up_disks([system.drive] + system.extra_drives)
 
-    spec = vmdb_spec(args.luks, args.playbook, args.playbook2, extra_vars)
+    ansible_vars = dict(system.ansible_vars)
+    ansible_vars["hostname"] = system.hostname
+    vmdb = vmdb_spec(system, ansible_vars)
     tmp = tempfile.mkdtemp()
     specfile = os.path.join(tmp, "spec.yaml")
-    if args.verbose:
-        yaml.dump(spec, stream=sys.stdout, indent=4)
+    if args.very_verbose:
+        yaml.dump(vmdb, stream=sys.stdout, indent=4)
     with open(specfile, "w") as f:
-        yaml.dump(spec, stream=f, indent=4)
+        yaml.dump(vmdb, stream=f, indent=4)
 
-    log(f"run vmdb2 to install on {args.device}")
+    log(f"run vmdb2 to install on {system.drive}")
     env = dict(os.environ)
     env["ANSIBLE_STDOUT_CALLBACK"] = "yaml"
     env["ANSIBLE_NOCOWS"] = "1"
@@ -322,12 +455,11 @@ def main():
         "vmdb2",
         f"--rootfs-tarball={args.cache}",
         f"--log={args.log}",
-        f"--image={args.device}",
+        f"--image={system.drive}",
         specfile,
     ]
     if verbose:
         argv.append("--verbose")
-
     run(argv, check=True, capture_output=True)
 
     log("cleanup")