Merge pull request 'liw/qemu' (#4) from liw/qemu into main

Reviewed-on: https://codeberg.org/ambient/ambient-run/pulls/4

11 files changed, 450 insertions, 10 deletions

diff --git a/Cargo.lock b/Cargo.lock
index a8cefda..7149f3d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -32,6 +32,7 @@ dependencies = [
  "subplot-build",
  "subplotlib",
  "tar",
+ "tempfile",
  "thiserror",
 ]
 
@@ -1235,9 +1236,9 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.7.0"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
 dependencies = [
  "cfg-if",
  "fastrand",
diff --git a/Cargo.toml b/Cargo.toml
index 4e6243a..46648ff 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ serde = { version = "1.0.182", features = ["derive"] }
 serde_yaml = "0.9.25"
 subplotlib = "0.8.0"
 tar = "0.4.40"
+tempfile = "3.8.0"
 thiserror = "1.0.44"
 
 [build-dependencies]
diff --git a/ambient-run.md b/ambient-run.md
index c5dc188..1690cce 100644
--- a/ambient-run.md
+++ b/ambient-run.md
@@ -172,6 +172,60 @@ image: /my/image.qcow2
 ## Building with ambient-run
 
 ### Smoke test build works
+
+_Requirement:_ I can do a simple build that just runs the shell
+command `echo hello, world` using `ambient-run`.
+
+_Justification:_ This is the simplest possible project build, and if
+it works, then at least the very fundamental parts of `ambient-run`
+work, and vice versa.
+
+_Stakeholder:_ Lars.
+
+~~~scenario
+given an installed ambient-run
+given file smoke/project.yaml from smoke-project.yaml
+given image file image.qcow2 specified for test suite
+when I run ambient-run build smoke/project.yaml --log hello.log
+then file hello.log contains "hello, world"
+~~~
+~~~{#smoke-project.yaml .file .yaml}
+source: .
+shell: |
+  #!/bin/bash
+  set -xeuo pipefail
+  echo hello, world
+image: image.qcow2
+~~~
+
+### Build gets source code
+
+_Requirement:_ The build gets a copy of the specified source code.
+
+_Justification:_ This is the simplest possible project build, and if
+it works, then at least the very fundamental parts of `ambient-run`
+work, and vice versa.
+
+_Stakeholder:_ Lars.
+
+~~~scenario
+given an installed ambient-run
+given file foo-project.yaml
+given file foo/README.md from foo-project.yaml
+given image file image.qcow2 specified for test suite
+when I run ambient-run build foo-project.yaml --log foo.log
+then file foo.log contains "README.md"
+~~~
+
+
+~~~{#foo-project.yaml .file .yaml}
+source: foo
+shell: |
+  #!/bin/bash
+  ls -l
+image: image.qcow2
+~~~
+
 ### Build is given dependencies
 ### Cache is persistent between builds
 ### Build gets the resources is demands
diff --git a/src/bin/ambient-run.rs b/src/bin/ambient-run.rs
index a2d3809..e1f1b16 100644
--- a/src/bin/ambient-run.rs
+++ b/src/bin/ambient-run.rs
@@ -2,6 +2,7 @@ use ambient_run::{
     config::{canonical, default_config_file, Config},
     error::AmbientRunError,
     project::Project,
+    qemu::Qemu,
     vdrive::VirtualDriveBuilder,
 };
 use clap::{Parser, Subcommand};
@@ -28,6 +29,7 @@ fn fallible_main() -> Result<(), AmbientRunError> {
         Command::Config(x) => x.run(&args, &config),
         Command::Project(x) => x.run(&args, &config),
         Command::VDrive(x) => x.run(&args, &config),
+        Command::Build(x) => x.run(&args, &config),
     }
 }
 
@@ -53,6 +55,7 @@ enum Command {
     Project(ProjectCommand),
     #[clap(name = "vdrive")]
     VDrive(VDriveCommand),
+    Build(BuildCommand),
 }
 
 #[derive(Debug, Parser)]
@@ -154,7 +157,7 @@ impl VDriveCreateCommand {
         if let Some(size) = self.size {
             builder = builder.size(size);
         }
-        builder.create()?;
+        builder.create(None)?;
         Ok(())
     }
 }
@@ -196,3 +199,27 @@ impl VDriveExtractCommand {
         Ok(())
     }
 }
+
+#[derive(Debug, Parser)]
+struct BuildCommand {
+    /// Project build specification.
+    filename: PathBuf,
+
+    /// Build log.
+    #[clap(long)]
+    log: Option<PathBuf>,
+}
+
+impl BuildCommand {
+    fn run(&self, _global: &Args, config: &Config) -> Result<(), AmbientRunError> {
+        let project = Project::load(&self.filename, config)?;
+        let mut qemu = Qemu::new(&project.image())
+            .with_shell(project.shell())
+            .with_source(&project.source());
+        if let Some(log) = &self.log {
+            qemu = qemu.with_log(log);
+        }
+        qemu.run()?;
+        Ok(())
+    }
+}
diff --git a/src/error.rs b/src/error.rs
index d3a03ed..0958d35 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -10,5 +10,8 @@ pub enum AmbientRunError {
     Project(#[from] crate::project::ProjectError),
 
     #[error(transparent)]
+    Qemu(#[from] crate::qemu::QemuError),
+
+    #[error(transparent)]
     VDrive(#[from] crate::vdrive::VirtualDriveError),
 }
diff --git a/src/lib.rs b/src/lib.rs
index 3e7428d..c29912e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,4 +5,5 @@
 pub mod config;
 pub mod error;
 pub mod project;
+pub mod qemu;
 pub mod vdrive;
diff --git a/src/project.rs b/src/project.rs
index b672867..8a182bc 100644
--- a/src/project.rs
+++ b/src/project.rs
@@ -7,12 +7,29 @@ use std::path::{Path, PathBuf};
 /// Per-project build instructions.
 #[derive(Debug, Deserialize, Serialize)]
 pub struct Project {
+    #[serde(skip)]
+    filename: PathBuf,
     source: PathBuf,
     shell: String,
     image: PathBuf,
 }
 
 impl Project {
+    // Directory where project build instructions were loaded from. If
+    // we can't figure that out, default to root directory.
+    fn directory(&self) -> &Path {
+        if let Some(parent) = self.filename.parent() {
+            parent
+        } else {
+            Path::new("/")
+        }
+    }
+
+    /// Return path relative to project build instruction directory.
+    pub fn join(&self, path: &Path) -> PathBuf {
+        self.directory().join(path)
+    }
+
     /// Load build instructions from named file.
     pub fn load(filename: &Path, config: &Config) -> Result<Self, ProjectError> {
         let no_image = PathBuf::from("/no/image/specified");
@@ -22,6 +39,7 @@ impl Project {
             no_image.clone()
         };
         let mut project = Self {
+            filename: filename.into(),
             source: PathBuf::from("."),
             shell: "".into(),
             image,
@@ -36,15 +54,11 @@ impl Project {
         Ok(project)
     }
 
-    /// Load named project file, update self with values.
-    pub fn add_from(&mut self, filename: &Path) -> Result<(), ProjectError> {
-        eprintln!("adding from {}", filename.display());
+    fn add_from(&mut self, filename: &Path) -> Result<(), ProjectError> {
         let bytes = std::fs::read(filename).map_err(|e| ProjectError::Open(filename.into(), e))?;
         let text = String::from_utf8_lossy(&bytes);
-        eprintln!("text: {:?}", text);
         let snippet: ProjectSnippet =
             serde_yaml::from_str(&text).map_err(|e| ProjectError::Yaml(filename.into(), e))?;
-        eprintln!("snippet: {:#?}", snippet);
         if let Some(x) = snippet.source {
             self.source = x;
         }
@@ -61,6 +75,21 @@ impl Project {
     pub fn as_yaml(&self) -> Result<String, ProjectError> {
         serde_yaml::to_string(self).map_err(ProjectError::AsYaml)
     }
+
+    /// Source directory.
+    pub fn source(&self) -> PathBuf {
+        self.join(self.source.as_ref())
+    }
+
+    /// VM image to use for this build.
+    pub fn image(&self) -> PathBuf {
+        self.image.clone()
+    }
+
+    /// Shell snippet to run to build the project.
+    pub fn shell(&self) -> &str {
+        self.shell.as_ref()
+    }
 }
 
 #[derive(Debug, Deserialize)]
@@ -90,3 +119,30 @@ pub enum ProjectError {
     #[error("no image specified by project or config")]
     NoImage,
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use std::fs::write;
+
+    #[test]
+    fn knows_its_directory() {
+        let config = Config::default();
+        let tempdir = tempfile::tempdir().expect("create temporary directory");
+        let filename = tempdir.path().join("project.yaml");
+        write(
+            &filename,
+            r#"shell: |
+  #!/bin/bash
+  echo hello
+image: nope
+source: yup
+"#,
+        )
+        .expect("write project spec file");
+        let p = Project::load(&filename, &config).expect("load build spec");
+        assert_eq!(p.directory(), tempdir.path());
+        assert_eq!(p.image(), Path::new("nope"));
+        assert_eq!(p.source(), tempdir.path().join("yup"));
+    }
+}
diff --git a/src/qemu.rs b/src/qemu.rs
new file mode 100644
index 0000000..c48213f
--- /dev/null
+++ b/src/qemu.rs
@@ -0,0 +1,272 @@
+//! Run QEMU.
+
+use crate::vdrive::{VirtualDrive, VirtualDriveBuilder, VirtualDriveError};
+
+use std::{
+    fs::{copy, write, File},
+    path::{Path, PathBuf},
+    process::{Command, Stdio},
+};
+
+const MAX_OUTPUT_SIZE: u64 = 1024 * 1024 * 1024;
+const OVMF_FD: &str = "/usr/share/ovmf/OVMF.fd";
+
+/// A QEMU runner.
+#[derive(Debug, Default)]
+pub struct Qemu {
+    image: PathBuf,
+    log: Option<PathBuf>,
+    source: PathBuf,
+    shell: Option<String>,
+}
+
+impl Qemu {
+    /// Create a new runner.
+    pub fn new(image: &Path) -> Self {
+        Self {
+            image: image.into(),
+            source: PathBuf::from("."),
+            ..Default::default()
+        }
+    }
+
+    /// Set log file.
+    pub fn with_log(mut self, log: &Path) -> Self {
+        self.log = Some(log.into());
+        self
+    }
+
+    /// Set source directory.
+    pub fn with_source(mut self, source: &Path) -> Self {
+        self.source = source.into();
+        self
+    }
+
+    /// Set script to run.
+    pub fn with_shell(mut self, shell: &str) -> Self {
+        self.shell = Some(shell.into());
+        self
+    }
+
+    /// Run QEMU in the specified way.
+    pub fn run(&self) -> Result<(), QemuError> {
+        eprintln!("qemu run");
+        let tmp = tempfile::tempdir().map_err(QemuError::TempDir)?;
+        let empty = tempfile::tempdir().map_err(QemuError::TempDir)?;
+
+        let image = tmp.path().join("vm.qcow2");
+        let vars = tmp.path().join("vars.fd");
+
+        eprintln!("copy image and vars");
+        copy(&self.image, &image).map_err(|e| QemuError::Copy(self.image.clone(), e))?;
+        copy(OVMF_FD, &vars).map_err(|e| QemuError::Copy(OVMF_FD.into(), e))?;
+
+        eprintln!("output drive");
+        let output_drive =
+            Self::create_tar_with_size(tmp.path().join("output"), empty.path(), MAX_OUTPUT_SIZE)?;
+
+        eprintln!("cache drive");
+        let cache_drive =
+            Self::create_tar_with_size(tmp.path().join("cache"), empty.path(), MAX_OUTPUT_SIZE)?;
+
+        eprintln!("deps drive");
+        let deps_drive = Self::create_tar(tmp.path().join("deps"), empty.path())?;
+
+        eprintln!("script");
+        let script = empty.path().join("ambient-script");
+        if let Some(shell) = &self.shell {
+            write(&script, shell).map_err(QemuError::Write)?;
+        }
+
+        eprintln!("source drive");
+        let source_drive = Self::create_tar_with_extra_file(
+            tmp.path().join("src"),
+            &self.source,
+            &script,
+            ".ambient-script",
+        )?;
+
+        let args = QemuArgs::default()
+            .with_valued_arg("-m", "16384")
+            .with_valued_arg("-smp", "cpus=4")
+            .with_valued_arg("-display", "none")
+            .with_valued_arg("-chardev", "stdio,id=serial0")
+            .with_valued_arg("-serial", "chardev:serial0")
+            .with_ipflash(0, OVMF_FD, true)
+            .with_ipflash(1, vars.to_str().unwrap(), false)
+            .with_qcow2(image.to_str().unwrap())
+            .with_raw(source_drive.filename(), true)
+            .with_raw(output_drive.filename(), false)
+            .with_raw(cache_drive.filename(), false)
+            .with_raw(deps_drive.filename(), true)
+            .with_arg("-nodefaults");
+
+        eprintln!("log file");
+        let log = Self::create_file(&self.log)?;
+
+        eprintln!("spawn qemu");
+        let child = Command::new("kvm")
+            .args(args.iter())
+            .stdin(Stdio::null())
+            .stdout(Stdio::from(log))
+            .stderr(Stdio::null())
+            .spawn()
+            .map_err(QemuError::Run)?;
+
+        eprintln!("wait for qemu");
+        let output = child.wait_with_output().map_err(QemuError::Run)?;
+        if output.status.success() {
+            eprintln!("kvm OK");
+            Ok(())
+        } else {
+            let out = String::from_utf8_lossy(&output.stdout);
+            let err = String::from_utf8_lossy(&output.stderr);
+            eprintln!("kvm ERROR\n{}\n{}", out, err);
+            Err(QemuError::Kvm)
+        }
+    }
+
+    fn create_file(filename: &Option<PathBuf>) -> Result<File, QemuError> {
+        let filename = if let Some(filename) = filename {
+            filename
+        } else {
+            Path::new("/dev/null")
+        };
+        let file = File::create(filename).map_err(|e| QemuError::Log(filename.into(), e))?;
+        Ok(file)
+    }
+
+    fn create_tar(tar_filename: PathBuf, dirname: &Path) -> Result<VirtualDrive, QemuError> {
+        let tar = VirtualDriveBuilder::default()
+            .filename(&tar_filename)
+            .root_directory(dirname)
+            .create(None)
+            .map_err(|e| QemuError::Tar(dirname.into(), e))?;
+        Ok(tar)
+    }
+
+    fn create_tar_with_extra_file(
+        tar_filename: PathBuf,
+        dirname: &Path,
+        extra: &Path,
+        name: &str,
+    ) -> Result<VirtualDrive, QemuError> {
+        let tar = VirtualDriveBuilder::default()
+            .filename(&tar_filename)
+            .root_directory(dirname)
+            .create(Some((extra, name)))
+            .map_err(|e| QemuError::Tar(dirname.into(), e))?;
+        Ok(tar)
+    }
+
+    fn create_tar_with_size(
+        tar_filename: PathBuf,
+        dirname: &Path,
+        size: u64,
+    ) -> Result<VirtualDrive, QemuError> {
+        let tar = VirtualDriveBuilder::default()
+            .filename(&tar_filename)
+            .root_directory(dirname)
+            .size(size)
+            .create(None)
+            .map_err(|e| QemuError::Tar(dirname.into(), e))?;
+        Ok(tar)
+    }
+}
+
+#[derive(Debug, Default)]
+struct QemuArgs {
+    args: Vec<String>,
+}
+
+impl QemuArgs {
+    fn with_arg(mut self, arg: &str) -> Self {
+        self.args.push(arg.into());
+        self
+    }
+
+    fn with_valued_arg(mut self, arg: &str, value: &str) -> Self {
+        self.args.push(arg.into());
+        self.args.push(value.into());
+        self
+    }
+
+    fn with_ipflash(mut self, unit: usize, path: &str, readonly: bool) -> Self {
+        self.args.push("-drive".into());
+        self.args.push(format!(
+            "if=pflash,format=raw,unit={},file={}{}",
+            unit,
+            path,
+            if readonly { ",readonly=on" } else { "" },
+        ));
+        self
+    }
+
+    fn with_qcow2(mut self, path: &str) -> Self {
+        self.args.push("-drive".into());
+        self.args
+            .push(format!("format=qcow2,if=virtio,file={}", path));
+        self
+    }
+
+    fn with_raw(mut self, path: &Path, readonly: bool) -> Self {
+        self.args.push("-drive".into());
+        self.args.push(format!(
+            "format=raw,if=virtio,file={}{}",
+            path.display(),
+            if readonly { ",readonly=on" } else { "" },
+        ));
+        self
+    }
+
+    fn iter(&self) -> impl Iterator<Item = &str> {
+        self.args.iter().map(|s| s.as_str())
+    }
+}
+
+/// Possible errors from running Qemu.
+#[allow(missing_docs)]
+#[derive(Debug, thiserror::Error)]
+pub enum QemuError {
+    #[error("failed to run QEMU")]
+    Run(#[source] std::io::Error),
+
+    #[error("failed to run QEMU")]
+    Kvm,
+
+    #[error("failed to create a temporary directory")]
+    TempDir(#[source] std::io::Error),
+
+    #[error("failed to copy to temporary directory: {0}")]
+    Copy(PathBuf, #[source] std::io::Error),
+
+    #[error("failed to write temporary file")]
+    Write(#[source] std::io::Error),
+
+    #[error("failed to open log file for writing")]
+    Log(PathBuf, #[source] std::io::Error),
+
+    #[error("failed to create a tar archive from {0}")]
+    Tar(PathBuf, #[source] VirtualDriveError),
+}
+
+#[cfg(test)]
+mod test {
+    use super::Qemu;
+    use std::path::Path;
+
+    #[test]
+    fn sets_image() {
+        let path = Path::new("/my/image.qcow2");
+        let qemu = Qemu::new(path);
+        assert_eq!(&qemu.image, &path);
+    }
+
+    #[test]
+    fn sets_log() {
+        let path = Path::new("/my/image.qcow2");
+        let log = Path::new("/my/log");
+        let qemu = Qemu::new(path).with_log(log);
+        assert_eq!(qemu.log, Some(log.into()));
+    }
+}
diff --git a/src/vdrive.rs b/src/vdrive.rs
index e86eb9d..12ec823 100644
--- a/src/vdrive.rs
+++ b/src/vdrive.rs
@@ -79,7 +79,7 @@ impl VirtualDriveBuilder {
     }
 
     /// Create a virtual drive.
-    pub fn create(self) -> Result<VirtualDrive, VirtualDriveError> {
+    pub fn create(self, extra: Option<(&Path, &str)>) -> Result<VirtualDrive, VirtualDriveError> {
         let filename = self.filename.expect("filename has been set");
         let file =
             File::create(&filename).map_err(|e| VirtualDriveError::Create(filename.clone(), e))?;
@@ -93,6 +93,11 @@ impl VirtualDriveBuilder {
                 .append_dir_all(".", &root)
                 .map_err(VirtualDriveError::CreateTar)?;
         }
+        if let Some((path, name)) = extra {
+            builder
+                .append_path_with_name(path, name)
+                .map_err(VirtualDriveError::CreateTar)?;
+        }
         Ok(VirtualDrive { filename })
     }
 
diff --git a/subplot/ambient-run.rs b/subplot/ambient-run.rs
index 508588c..0b5f097 100644
--- a/subplot/ambient-run.rs
+++ b/subplot/ambient-run.rs
@@ -5,7 +5,7 @@ use subplotlib::steplibrary::datadir::Datadir;
 use subplotlib::steplibrary::files::Files;
 use subplotlib::steplibrary::runcmd::Runcmd;
 
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 #[derive(Debug, Default)]
 struct SubplotContext {}
@@ -30,6 +30,22 @@ fn install_ambient_run(context: &ScenarioContext) {
 
 #[step]
 #[context(SubplotContext)]
+#[context(Datadir)]
+fn copy_image_file(context: &ScenarioContext, filename: &Path) {
+    let image = PathBuf::from(std::env::var_os("IMAGE").expect("exected IMAGE to be set"));
+    let filename = context.with_mut(
+        |context: &mut Datadir| {
+            Ok(context
+                .canonicalise_filename(filename)
+                .expect("canonical filename for image file copy"))
+        },
+        false,
+    )?;
+    std::os::unix::fs::symlink(image, filename).expect("symlink $IMAGE to test dir");
+}
+
+#[step]
+#[context(SubplotContext)]
 #[context(Runcmd)]
 #[context(Files)]
 fn stdout_matches_yaml(context: &ScenarioContext, file: SubplotDataFile) {
diff --git a/subplot/ambient-run.yaml b/subplot/ambient-run.yaml
index 50d04f9..f44eb29 100644
--- a/subplot/ambient-run.yaml
+++ b/subplot/ambient-run.yaml
@@ -2,6 +2,10 @@
   impl:
     rust:
       function: install_ambient_run
+- given: image file {filename} specified for test suite
+  impl:
+    rust:
+      function: copy_image_file
 - then: stdout, as YAML, matches file {file:file}
   impl:
     rust: