Merge branch 'meta' into 'main'

Meta

See merge request larswirzenius/obnam!35

4 files changed, 134 insertions, 24 deletions

diff --git a/Cargo.toml b/Cargo.toml
index 2d6f11e..652b727 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,7 @@ edition = "2018"
 anyhow = "1"
 bytes = "0.5"
 indicatif = "0.15"
+libc = "0.2"
 log = "0.4"
 pretty_env_logger = "0.4"
 reqwest = { version = "0.10", features = ["blocking", "json"]}
diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs
index f2724c9..2b62ef5 100644
--- a/src/cmd/restore.rs
+++ b/src/cmd/restore.rs
@@ -3,9 +3,12 @@ use crate::client::ClientConfig;
 use crate::fsentry::{FilesystemEntry, FilesystemKind};
 use crate::generation::Generation;
 use indicatif::{ProgressBar, ProgressStyle};
-use log::{debug, info};
+use libc::{fchmod, futimens, timespec};
+use log::{debug, error, info};
 use std::fs::File;
 use std::io::prelude::*;
+use std::io::Error;
+use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
 use structopt::StructOpt;
 use tempfile::NamedTempFile;
@@ -36,10 +39,15 @@ pub fn restore(config: &Path, gen_id: &str, to: &Path) -> anyhow::Result<()> {
     info!("downloaded generation to {}", dbname.display());
 
     let gen = Generation::open(&dbname)?;
-    println!("file count: {}", gen.file_count());
-    let progress = create_progress_bar(gen.file_count());
+    info!("restore file count: {}", gen.file_count());
+    let progress = create_progress_bar(gen.file_count(), false);
     for (fileid, entry) in gen.files()? {
-        restore_generation(&client, &gen, fileid, entry, &to, &progress)?;
+        restore_generation(&client, &gen, fileid, &entry, &to, &progress)?;
+    }
+    for (_, entry) in gen.files()? {
+        if entry.is_dir() {
+            restore_directory_metadata(&entry, &to)?;
+        }
     }
     progress.finish();
 
@@ -69,21 +77,14 @@ fn restore_generation(
     client: &BackupClient,
     gen: &Generation,
     fileid: u64,
-    entry: FilesystemEntry,
+    entry: &FilesystemEntry,
     to: &Path,
     progress: &ProgressBar,
 ) -> anyhow::Result<()> {
     progress.set_message(&format!("{}", entry.path().display()));
     progress.inc(1);
 
-    let path = if entry.path().is_absolute() {
-        entry.path().strip_prefix("/")?
-    } else {
-        entry.path()
-    };
-    let to = to.join(path);
-    debug!("  to: {}", to.display());
-
+    let to = restored_path(entry, to)?;
     match entry.kind() {
         FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?,
         FilesystemKind::Directory => restore_directory(&to)?,
@@ -97,12 +98,33 @@ fn restore_directory(path: &Path) -> anyhow::Result<()> {
     Ok(())
 }
 
+fn restore_directory_metadata(entry: &FilesystemEntry, to: &Path) -> anyhow::Result<()> {
+    let to = restored_path(entry, to)?;
+    match entry.kind() {
+        FilesystemKind::Directory => restore_metadata(&to, entry)?,
+        _ => panic!(
+            "restore_directory_metadata called with non-directory {:?}",
+            entry,
+        ),
+    }
+    Ok(())
+}
+
+fn restored_path(entry: &FilesystemEntry, to: &Path) -> anyhow::Result<PathBuf> {
+    let path = if entry.path().is_absolute() {
+        entry.path().strip_prefix("/")?
+    } else {
+        entry.path()
+    };
+    Ok(to.join(path))
+}
+
 fn restore_regular(
     client: &BackupClient,
     gen: &Generation,
     path: &Path,
     fileid: u64,
-    _entry: &FilesystemEntry,
+    entry: &FilesystemEntry,
 ) -> anyhow::Result<()> {
     debug!("restoring regular {}", path.display());
     let parent = path.parent().unwrap();
@@ -114,13 +136,56 @@ fn restore_regular(
             let chunk = client.fetch_chunk(&chunkid)?;
             file.write_all(chunk.data())?;
         }
+        restore_metadata(path, entry)?;
     }
     debug!("restored regular {}", path.display());
     Ok(())
 }
 
-fn create_progress_bar(file_count: u64) -> ProgressBar {
-    let progress = ProgressBar::new(file_count);
+fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> anyhow::Result<()> {
+    debug!("restoring metadata for {}", entry.path().display());
+
+    let handle = File::open(path)?;
+
+    let atime = timespec {
+        tv_sec: entry.atime(),
+        tv_nsec: entry.atime_ns(),
+    };
+    let mtime = timespec {
+        tv_sec: entry.mtime(),
+        tv_nsec: entry.mtime_ns(),
+    };
+    let times = [atime, mtime];
+    let times: *const timespec = &times[0];
+
+    // We have to use unsafe here to be able call the libc functions
+    // below.
+    unsafe {
+        let fd = handle.as_raw_fd(); // FIXME: needs to NOT follow symlinks
+
+        debug!("fchmod");
+        if fchmod(fd, entry.mode()) == -1 {
+            let error = Error::last_os_error();
+            error!("fchmod failed on {}", path.display());
+            return Err(error.into());
+        }
+
+        debug!("futimens");
+        if futimens(fd, times) == -1 {
+            let error = Error::last_os_error();
+            error!("futimens failed on {}", path.display());
+            return Err(error.into());
+        }
+    }
+    Ok(())
+}
+
+fn create_progress_bar(file_count: u64, verbose: bool) -> ProgressBar {
+    let progress = if verbose {
+        ProgressBar::new(file_count)
+    } else {
+        ProgressBar::hidden()
+    };
     let parts = vec![
         "{wide_bar}",
         "elapsed: {elapsed}",
diff --git a/src/fsentry.rs b/src/fsentry.rs
index b8be212..fd09ea2 100644
--- a/src/fsentry.rs
+++ b/src/fsentry.rs
@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 use std::fs::{FileType, Metadata};
+use std::os::linux::fs::MetadataExt;
 use std::path::{Path, PathBuf};
 
 /// A file system entry.
@@ -16,15 +17,33 @@ pub struct FilesystemEntry {
     kind: FilesystemKind,
     path: PathBuf,
     len: u64,
+
+    // 16 bits should be enough for a Unix mode_t.
+    // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_stat.h.html
+    //  However, it's 32 bits on Linux, so that's what we store.
+    mode: u32,
+
+    // Linux can store file system time stamps in nanosecond
+    // resolution. We store them as two 64-bit integers.
+    mtime: i64,
+    mtime_ns: i64,
+    atime: i64,
+    atime_ns: i64,
 }
 
 #[allow(clippy::len_without_is_empty)]
 impl FilesystemEntry {
     pub fn from_metadata(path: &Path, meta: &Metadata) -> Self {
-        let path = path.to_path_buf();
-        let kind = FilesystemKind::from_file_type(meta.file_type());
-        let len = meta.len();
-        Self { path, kind, len }
+        Self {
+            path: path.to_path_buf(),
+            kind: FilesystemKind::from_file_type(meta.file_type()),
+            len: meta.len(),
+            mode: meta.st_mode(),
+            mtime: meta.st_mtime(),
+            mtime_ns: meta.st_mtime_nsec(),
+            atime: meta.st_atime(),
+            atime_ns: meta.st_atime_nsec(),
+        }
     }
 
     pub fn kind(&self) -> FilesystemKind {
@@ -38,6 +57,30 @@ impl FilesystemEntry {
     pub fn len(&self) -> u64 {
         self.len
     }
+
+    pub fn mode(&self) -> u32 {
+        self.mode
+    }
+
+    pub fn atime(&self) -> i64 {
+        self.atime
+    }
+
+    pub fn atime_ns(&self) -> i64 {
+        self.atime_ns
+    }
+
+    pub fn mtime(&self) -> i64 {
+        self.mtime
+    }
+
+    pub fn mtime_ns(&self) -> i64 {
+        self.mtime_ns
+    }
+
+    pub fn is_dir(&self) -> bool {
+        self.kind() == FilesystemKind::Directory
+    }
 }
 
 /// Different types of file system entries.
diff --git a/subplot/data.py b/subplot/data.py
index c4cb0a7..9eadd3f 100644
--- a/subplot/data.py
+++ b/subplot/data.py
@@ -28,7 +28,8 @@ def _create_manifest_of_directory(ctx, dirname=None, manifest=None):
     runcmd_get_exit_code = globals()["runcmd_get_exit_code"]
     runcmd_get_stdout = globals()["runcmd_get_stdout"]
 
-    runcmd_run(ctx, ["summain", dirname])
+    logging.info(f"creating manifest for {dirname} in {manifest}")
+    runcmd_run(ctx, ["find", "-exec", "summain", "{}", "+"], cwd=dirname)
     assert runcmd_get_exit_code(ctx) == 0
     stdout = runcmd_get_stdout(ctx)
     open(manifest, "w").write(stdout)
@@ -38,7 +39,7 @@ def files_match(ctx, first=None, second=None):
     assert_eq = globals()["assert_eq"]
 
     f = open(first).read()
-    s = open(first).read()
-    logging.debug(f"files_match: f={f!r}")
-    logging.debug(f"files_match: s={s!r}")
+    s = open(second).read()
+    logging.debug(f"files_match: f:\n{f}")
+    logging.debug(f"files_match: s:\n{s}")
     assert_eq(f, s)