feat: replace options with a YAML specification file

Files can be kept in git. Command line invocation are less convenient.

Also: discard all the drives, allow extra LVs to be created, and
refactor the code to be easier to follow.

Sponsored-by: author

1 files changed, 285 insertions, 153 deletions

diff --git a/v-i b/v-i
index c1957fa..2f613a7 100755
--- a/v-i
+++ b/v-i
@@ -82,8 +82,8 @@ def is_luks(path):
     return p.returncode == 0
 
 
-def clean_up_disks(device):
-    log("clean up disks from old installs")
+def clean_up_disks(drives):
+    log(f"clean system LVM2 and drives from everything: {drives}")
 
     mounts = mount_points()
     pvs = physical_volumes()
@@ -115,118 +115,223 @@ def clean_up_disks(device):
             log(f"open LUKS volume {mapping} (just in case it is one)")
             run(["cryptsetup", "close", mapping], check=False, capture_output=True)
 
-    log(f"blkdiscard {device}")
-    run(["blkdiscard", "--force", device], check=True, capture_output=True)
+    for drive in drives:
+        log(f"blkdiscard {drive}")
+        run(
+            ["blkdiscard", "--force", drive],
+            check=True,
+            capture_output=True,
+        )
 
 
-def vmdb_spec(cryptsetup_password, playbook, playbook2, extra_vars):
-    device = "{{ image }}"
-    spec = {
-        "steps": [
-            {
-                "mklabel": "gpt",
-                "device": device,
-            },
-            {
-                "mkpart": "primary",
-                "device": device,
-                "start": "0%",
-                "end": "500M",
-                "tag": "efi",
-            },
-            {
-                "mkpart": "primary",
-                "device": device,
-                "start": "500M",
-                "end": "1G",
-                "tag": "boot",
-            },
-        ]
+def mklabel(device):
+    return {
+        # Create a GPT partition table. It works well for all modern
+        # systems.
+        "mklabel": "gpt",
+        "device": device,
+    }
+
+
+def mkpart(device, tag, start, end):
+    return {
+        "mkpart": "primary",
+        "device": device,
+        "start": start,
+        "end": end,
+        "tag": tag,
+    }
+
+
+def mkfs(tag, fstype):
+    return {
+        "mkfs": fstype,
+        "partition": tag,
+    }
+
+
+def cryptsetup(tag, password, name):
+    return {
+        "cryptsetup": "cryptsetup0",
+        "password": password,
+        "name": name,
     }
 
+
+def vgcreate(tag, drives):
+    return {
+        "vgcreate": tag,
+        "physical": drives,
+    }
+
+
+def lvcreate(vg, name, size):
+    return {
+        "lvcreate": vg,
+        "name": name,
+        "size": size,
+    }
+
+
+def mount(tag, dirname=None, mount_on=None):
+    d = {"mount": tag}
+    if dirname is not None:
+        d["dirname"] = dirname
+    if mount_on is not None:
+        d["mount-on"] = mount_on
+    return d
+
+
+def unpack_rootfs(tag):
+    return {
+        "unpack-rootfs": tag,
+    }
+
+
+def cache_rootfs(tag):
+    return {
+        "cache-rootfs": tag,
+        "unless": "rootfs_unpacked",
+    }
+
+
+def debootstrap(tag):
+    return {
+        "debootstrap": "bullseye",
+        "mirror": "http://deb.debian.org/debian",
+        "target": tag,
+        "unless": "rootfs_unpacked",
+    }
+
+
+def apt(tag, packages, unless=True):
+    d = {
+        "apt": "install",
+        "packages": [
+            "console-setup",
+            "dosfstools",
+            "linux-image-amd64",
+            "locales-all",
+            "lvm2",
+            "psmisc",
+            "python3",
+            "ssh",
+            "strace",
+        ],
+        "tag": tag,
+    }
+    if unless:
+        d["unless"] = "rootfs_unpacked"
+    return d
+
+
+def virtual_filesystems(tag):
+    return {
+        "virtual-filesystems": "root",
+    }
+
+
+def fstab(tag):
+    return {
+        "fstab": tag,
+    }
+
+
+def grub(device, root, efi):
+    return {
+        "grub": "uefi",
+        "tag": root,
+        "efi": efi,
+        "quiet": True,
+        "image-dev": device,
+    }
+
+
+def vmdb_spec(system, ansible_vars):
+    device = "{{ image }}"
+    steps = [
+        mklabel(device),
+        # Create a partition for UEFI to boot from.
+        mkpart(device, "efi", "0%", "500M"),
+        # Create a separate /boot partition, in case we want to have LUKS.
+        mkpart(device, "boot", "500M", "1G"),
+        # Format the EFI partition. This MUST be vfat.
+        mkfs("efi", "vfat"),
+        # Format /boot. This is conventionally ext2. This file system
+        # gets very little I/O, but it MUST be supported by GRUB, so
+        # ext2 seems like a nice, safe choice.
+        mkfs("boot", "ext2"),
+    ]
+
     # Set up pv0 for lvm2, either encrypted or cleartext.
-    if cryptsetup_password:
-        spec["steps"].extend(
+    if system.luks:
+        steps.extend(
             [
-                {
-                    "mkpart": "primary",
-                    "device": device,
-                    "start": "1G",
-                    "end": "100%",
-                    "tag": "cryptsetup0",
-                },
-                {
-                    "cryptsetup": "cryptsetup0",
-                    "password": cryptsetup_password,
-                    "name": "pv0",
-                },
+                mkpart(device, "cryptsetup0", "1G", "100%"),
+                cryptsetup("cryptsetup0", system.luks, "pv0"),
             ]
         )
+        for (i, drive) in enumerate(system.extra_drives):
+            steps.append(cryptsetup(f"cryptsetuo{i+1}", system.luks, f"pv{i+1}"))
     else:
-        spec["steps"].extend(
+        steps.append(mkpart(device, "pv0", "1G", "100%"))
+        for (i, drive) in enumerate(system.extra_drives):
+            steps.extend(
+                [
+                    mklabel(drive),
+                    mkpart(drive, f"pv{i+1}", "0%", "100%"),
+                ]
+            )
+
+    # Create file systems and install Debian.
+    steps.extend(
+        [
+            # Create an LVM2 volume group using pv0, which we create
+            # earlier. At this point, if LUKS is used, pv0 is the unlocked,
+            # open, cleartext block device.
+            vgcreate(
+                "vg0",
+                [f"pv{i}" for i in range(len([system.drive] + system.extra_drives))],
+            ),
+            # Create a 20 gigabyte LV for the root file system. That's big
+            # enough for a desktop system.
+            lvcreate("vg0", "root", "20G"),
+            # format the root file system. This gets a fair bit of use, so
+            # ext4 seems like a safe choice. If you wanted another file
+            # system, sorry.
+            mkfs("root", "ext4"),
+            # Mount the root file system.
+            mount("root"),
+            # Mount /boot on top of the root file system.
+            mount("boot", dirname="/boot", mount_on="root"),
+            # Mount /boot/efi.
+            mount("efi", dirname="/boot/efi", mount_on="boot"),
+        ]
+    )
+
+    # Add any additional LVs.
+    for lv in system.extra_lvs:
+        steps.extend(
             [
-                {
-                    "mkpart": "primary",
-                    "device": device,
-                    "start": "1G",
-                    "end": "100%",
-                    "tag": "pv0",
-                },
+                lvcreate("vg0", lv["name"], lv["size"]),
+                mkfs(lv["name"], "ext4"),
             ]
         )
 
-    # Create file systems and install Debian.
-    spec["steps"].extend(
+    steps.extend(
         [
-            {
-                "mkfs": "vfat",
-                "partition": "efi",
-            },
-            {
-                "mkfs": "ext2",
-                "partition": "boot",
-            },
-            {
-                "vgcreate": "vg0",
-                "physical": ["pv0"],
-            },
-            {
-                "lvcreate": "vg0",
-                "name": "root",
-                "size": "10G",
-            },
-            {
-                "mkfs": "ext4",
-                "partition": "root",
-            },
-            {
-                "mount": "root",
-            },
-            {
-                "mount": "boot",
-                "dirname": "/boot",
-                "mount-on": "root",
-            },
-            {
-                "mount": "efi",
-                "dirname": "/boot/efi",
-                "mount-on": "boot",
-            },
-            {
-                "unpack-rootfs": "root",
-            },
-            {
-                "debootstrap": "bullseye",
-                "mirror": "http://deb.debian.org/debian",
-                "target": "root",
-                "unless": "rootfs_unpacked",
-            },
-            {
-                "apt": "install",
-                "packages": [
+            # If we have a cached version of the installed system, unpack
+            # it now. Otherwise do nothing. Note that if you make any
+            # changes to the steps marked "unless: rootfs_unpacked", you
+            # have to remember to manually remove the cache file. v-i or
+            # vmdb2 won't do that automatically for you.
+            unpack_rootfs("root"),
+            debootstrap("root"),
+            apt(
+                "root",
+                [
                     "console-setup",
                     "dosfstools",
-                    # "ifupdown",
                     "linux-image-amd64",
                     "locales-all",
                     "lvm2",
@@ -235,84 +340,112 @@ def vmdb_spec(cryptsetup_password, playbook, playbook2, extra_vars):
                     "ssh",
                     "strace",
                 ],
-                "tag": "root",
-                "unless": "rootfs_unpacked",
-            },
-            {
-                "cache-rootfs": "root",
-                "unless": "rootfs_unpacked",
-            },
-            {
-                # This MUST be after the debootstrap step.
-                "virtual-filesystems": "root",
-            },
-            {
-                "fstab": "root",
-            },
-            {
-                # These MUST come after the fstab step so that they add the
-                # crypttab in the initramfs.
-                "apt": "install",
-                "packages": [
+            ),
+            # If we didn't unpack an existing cache archive, make one now.
+            # Otherwise, skip this step.
+            cache_rootfs("root"),
+            # This MUST be after the debootstrap step.
+            virtual_filesystems("root"),
+            # Create /etc/fstab (and, if LUKS is used, /etc/crypttab).
+            fstab("root"),
+            # These MUST come after the fstab step so that they add the
+            # crypttab in the initramfs. We install them regardless of
+            # whether LUKS is used: they're harmless if LUKS isn't used.
+            apt(
+                "root",
+                [
                     "cryptsetup",
                     "cryptsetup-initramfs",
                 ],
-                "tag": "root",
-            },
-            {
-                # This also MUST come outside the rootfs caching, as it install
-                # things outside the file systems.
-                "grub": "uefi",
-                "tag": "root",
-                "efi": "efi",
-                "quiet": True,
-                "image-dev": device,
-            },
+                unless=False,
+            ),
+            # This also MUST come outside the rootfs caching, as it install
+            # things outside the file systems, and those won't be in the
+            # cache.
+            grub(device, "root", "efi"),
         ]
     )
 
     # If playbooks have been specified, add ansible steps.
-    for p in (playbook, playbook2):
+    for p in ["std.yml"] + system.extra_playbooks:
         if p:
-            spec["steps"].append(
-                {"ansible": "root", "playbook": p, "extra_vars": extra_vars}
-            )
+            steps.append({"ansible": "root", "playbook": p, "extra_vars": ansible_vars})
+
+    return {"steps": steps}
+
+
+class SystemSpec:
+    def __init__(self, filename):
+        REQUIRED = "required"
+        self._obj = {
+            "hostname": REQUIRED,
+            "drive": REQUIRED,
+            "extra_drives": [],
+            "extra_lvs": [],
+            "extra_playbooks": [],
+            "ansible_vars": {},
+            "luks": None,
+        }
+        with open(filename) as f:
+            obj = yaml.safe_load(f)
+
+        # Check for unknown keys.
+        for key in obj:
+            if key not in self._obj:
+                sys.exit(f"spec has unknown key: {key}")
+
+        # Check for missing required keys.
+        for key in self._obj:
+            if self._obj[key] == REQUIRED and key not in obj:
+                sys.exit(f"spec lacks required key {key}")
+
+        # Check for types of values.
+        for key in self._obj:
+            if key in obj:
+                e = type(self._obj[key])
+                a = type(obj[key])
+                if a != e:
+                    sys.exit(f"spec key {key} has unexpected type {a}, wanted {e}")
 
-    return spec
+        self._obj.update(obj)
+
+        for key in self._obj:
+            setattr(self, key, self._obj[key])
+        del self._obj
+
+    def __repr__(self):
+        r = {key: getattr(self, key) for key in dir(self) if not key.startswith("_")}
+        return repr(r)
 
 
 def main():
     p = argparse.ArgumentParser()
     p.add_argument("--verbose", action="store_true")
+    p.add_argument("--very-verbose", action="store_true")
     p.add_argument("--log", default="install.log")
     p.add_argument("--cache", default="cache.tar.gz")
-    p.add_argument("--playbook", default="std.yml")
-    p.add_argument("--playbook2")
-    p.add_argument("--vars")
-    p.add_argument("--luks")
-    p.add_argument("device")
+    p.add_argument("spec")
     args = p.parse_args()
 
-    if args.verbose:
-        global verbose
-        verbose = args.verbose
+    global verbose
+    verbose = args.verbose
 
-    extra_vars = {}
-    if args.vars:
-        with open(args.vars) as f:
-            extra_vars = yaml.safe_load(f)
+    system = SystemSpec(args.spec)
+    log(f"spec: {system!r}")
 
-    clean_up_disks(args.device)
+    clean_up_disks([system.drive] + system.extra_drives)
 
-    spec = vmdb_spec(args.luks, args.playbook, args.playbook2, extra_vars)
+    ansible_vars = dict(system.ansible_vars)
+    ansible_vars["hostname"] = system.hostname
+    vmdb = vmdb_spec(system, ansible_vars)
     tmp = tempfile.mkdtemp()
     specfile = os.path.join(tmp, "spec.yaml")
-    if args.verbose:
-        yaml.dump(spec, stream=sys.stdout, indent=4)
+    if args.very_verbose:
+        yaml.dump(vmdb, stream=sys.stdout, indent=4)
     with open(specfile, "w") as f:
-        yaml.dump(spec, stream=f, indent=4)
+        yaml.dump(vmdb, stream=f, indent=4)
 
-    log(f"run vmdb2 to install on {args.device}")
+    log(f"run vmdb2 to install on {system.drive}")
     env = dict(os.environ)
     env["ANSIBLE_STDOUT_CALLBACK"] = "yaml"
     env["ANSIBLE_NOCOWS"] = "1"
@@ -322,12 +455,11 @@ def main():
         "vmdb2",
         f"--rootfs-tarball={args.cache}",
         f"--log={args.log}",
-        f"--image={args.device}",
+        f"--image={system.drive}",
         specfile,
     ]
     if verbose:
         argv.append("--verbose")
-
     run(argv, check=True, capture_output=True)
 
     log("cleanup")