From 02028a3239f45ada950e43a9fb34e23c6a8ccbed Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 11 Dec 2020 19:08:40 +0200 Subject: store mode in fsentry --- src/fsentry.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/fsentry.rs b/src/fsentry.rs index b8be212..7adf610 100644 --- a/src/fsentry.rs +++ b/src/fsentry.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; use std::fs::{FileType, Metadata}; +use std::os::linux::fs::MetadataExt; use std::path::{Path, PathBuf}; /// A file system entry. @@ -16,6 +17,11 @@ pub struct FilesystemEntry { kind: FilesystemKind, path: PathBuf, len: u64, + + // 16 bits should be enough for a Unix mode_t. + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_stat.h.html + // However, it's 32 bits on Linux, so that's what we store. + mode: u32, } #[allow(clippy::len_without_is_empty)] @@ -24,7 +30,13 @@ impl FilesystemEntry { let path = path.to_path_buf(); let kind = FilesystemKind::from_file_type(meta.file_type()); let len = meta.len(); - Self { path, kind, len } + let mode = meta.st_mode(); + Self { + path, + kind, + len, + mode, + } } pub fn kind(&self) -> FilesystemKind { -- cgit v1.2.1 From 01b9765db0feb3f1f36eec4d71489ccbaa6496b5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 11 Dec 2020 19:13:55 +0200 Subject: add timestamps to fsentry --- src/fsentry.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/fsentry.rs b/src/fsentry.rs index 7adf610..aae1475 100644 --- a/src/fsentry.rs +++ b/src/fsentry.rs @@ -22,20 +22,27 @@ pub struct FilesystemEntry { // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_stat.h.html // However, it's 32 bits on Linux, so that's what we store. mode: u32, + + // Linux can store file system time stamps in nanosecond + // resolution. We store them as two 64-bit integers. + mtime: i64, + mtime_ns: i64, + atime: i64, + atime_ns: i64, } #[allow(clippy::len_without_is_empty)] impl FilesystemEntry { pub fn from_metadata(path: &Path, meta: &Metadata) -> Self { - let path = path.to_path_buf(); - let kind = FilesystemKind::from_file_type(meta.file_type()); - let len = meta.len(); - let mode = meta.st_mode(); Self { - path, - kind, - len, - mode, + path: path.to_path_buf(), + kind: FilesystemKind::from_file_type(meta.file_type()), + len: meta.len(), + mode: meta.st_mode(), + mtime: meta.st_mtime(), + mtime_ns: meta.st_mtime_nsec(), + atime: meta.st_atime(), + atime_ns: meta.st_atime_nsec(), } } -- cgit v1.2.1 From 2035df9b6bad93a88fba7cd572d9939d1d6fa18d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 11 Dec 2020 19:22:44 +0200 Subject: fix: how manifests are created and compared --- subplot/data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/subplot/data.py b/subplot/data.py index c4cb0a7..9eadd3f 100644 --- a/subplot/data.py +++ b/subplot/data.py @@ -28,7 +28,8 @@ def _create_manifest_of_directory(ctx, dirname=None, manifest=None): runcmd_get_exit_code = globals()["runcmd_get_exit_code"] runcmd_get_stdout = globals()["runcmd_get_stdout"] - runcmd_run(ctx, ["summain", dirname]) + logging.info(f"creating manifest for {dirname} in {manifest}") + runcmd_run(ctx, ["find", "-exec", "summain", "{}", "+"], cwd=dirname) assert runcmd_get_exit_code(ctx) == 0 stdout = runcmd_get_stdout(ctx) open(manifest, "w").write(stdout) @@ -38,7 +39,7 @@ def files_match(ctx, first=None, second=None): assert_eq = globals()["assert_eq"] f = open(first).read() - s = open(first).read() - logging.debug(f"files_match: f={f!r}") - logging.debug(f"files_match: s={s!r}") + s = open(second).read() + logging.debug(f"files_match: f:\n{f}") + logging.debug(f"files_match: s:\n{s}") assert_eq(f, s) -- cgit v1.2.1 From 3d4e92075773b6f3034251ad63c4d1e4985352be Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 11 Dec 2020 20:14:55 +0200 Subject: feat: restore metadata as well as file contentents --- Cargo.toml | 1 + src/cmd/restore.rs | 97 +++++++++++++++++++++++++++++++++++++++++++++--------- src/fsentry.rs | 24 ++++++++++++++ 3 files changed, 106 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2d6f11e..652b727 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" anyhow = "1" bytes = "0.5" indicatif = "0.15" +libc = "0.2" log = "0.4" pretty_env_logger = "0.4" reqwest = { version = "0.10", features = ["blocking", "json"]} diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index f2724c9..2b62ef5 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -3,9 +3,12 @@ use crate::client::ClientConfig; use crate::fsentry::{FilesystemEntry, FilesystemKind}; use crate::generation::Generation; use indicatif::{ProgressBar, ProgressStyle}; -use log::{debug, info}; +use libc::{fchmod, futimens, timespec}; +use log::{debug, error, info}; use std::fs::File; use std::io::prelude::*; +use std::io::Error; +use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use structopt::StructOpt; use tempfile::NamedTempFile; @@ -36,10 +39,15 @@ pub fn restore(config: &Path, gen_id: &str, to: &Path) -> anyhow::Result<()> { info!("downloaded generation to {}", dbname.display()); let gen = Generation::open(&dbname)?; - println!("file count: {}", gen.file_count()); - let progress = create_progress_bar(gen.file_count()); + info!("restore file count: {}", gen.file_count()); + let progress = create_progress_bar(gen.file_count(), false); for (fileid, entry) in gen.files()? { - restore_generation(&client, &gen, fileid, entry, &to, &progress)?; + restore_generation(&client, &gen, fileid, &entry, &to, &progress)?; + } + for (_, entry) in gen.files()? { + if entry.is_dir() { + restore_directory_metadata(&entry, &to)?; + } } progress.finish(); @@ -69,21 +77,14 @@ fn restore_generation( client: &BackupClient, gen: &Generation, fileid: u64, - entry: FilesystemEntry, + entry: &FilesystemEntry, to: &Path, progress: &ProgressBar, ) -> anyhow::Result<()> { progress.set_message(&format!("{}", entry.path().display())); progress.inc(1); - let path = if entry.path().is_absolute() { - entry.path().strip_prefix("/")? - } else { - entry.path() - }; - let to = to.join(path); - debug!(" to: {}", to.display()); - + let to = restored_path(entry, to)?; match entry.kind() { FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?, FilesystemKind::Directory => restore_directory(&to)?, @@ -97,12 +98,33 @@ fn restore_directory(path: &Path) -> anyhow::Result<()> { Ok(()) } +fn restore_directory_metadata(entry: &FilesystemEntry, to: &Path) -> anyhow::Result<()> { + let to = restored_path(entry, to)?; + match entry.kind() { + FilesystemKind::Directory => restore_metadata(&to, entry)?, + _ => panic!( + "restore_directory_metadata called with non-directory {:?}", + entry, + ), + } + Ok(()) +} + +fn restored_path(entry: &FilesystemEntry, to: &Path) -> anyhow::Result { + let path = if entry.path().is_absolute() { + entry.path().strip_prefix("/")? + } else { + entry.path() + }; + Ok(to.join(path)) +} + fn restore_regular( client: &BackupClient, gen: &Generation, path: &Path, fileid: u64, - _entry: &FilesystemEntry, + entry: &FilesystemEntry, ) -> anyhow::Result<()> { debug!("restoring regular {}", path.display()); let parent = path.parent().unwrap(); @@ -114,13 +136,56 @@ fn restore_regular( let chunk = client.fetch_chunk(&chunkid)?; file.write_all(chunk.data())?; } + restore_metadata(path, entry)?; } debug!("restored regular {}", path.display()); Ok(()) } -fn create_progress_bar(file_count: u64) -> ProgressBar { - let progress = ProgressBar::new(file_count); +fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> anyhow::Result<()> { + debug!("restoring metadata for {}", entry.path().display()); + + let handle = File::open(path)?; + + let atime = timespec { + tv_sec: entry.atime(), + tv_nsec: entry.atime_ns(), + }; + let mtime = timespec { + tv_sec: entry.mtime(), + tv_nsec: entry.mtime_ns(), + }; + let times = [atime, mtime]; + let times: *const timespec = ×[0]; + + // We have to use unsafe here to be able call the libc functions + // below. + unsafe { + let fd = handle.as_raw_fd(); // FIXME: needs to NOT follow symlinks + + debug!("fchmod"); + if fchmod(fd, entry.mode()) == -1 { + let error = Error::last_os_error(); + error!("fchmod failed on {}", path.display()); + return Err(error.into()); + } + + debug!("futimens"); + if futimens(fd, times) == -1 { + let error = Error::last_os_error(); + error!("futimens failed on {}", path.display()); + return Err(error.into()); + } + } + Ok(()) +} + +fn create_progress_bar(file_count: u64, verbose: bool) -> ProgressBar { + let progress = if verbose { + ProgressBar::new(file_count) + } else { + ProgressBar::hidden() + }; let parts = vec![ "{wide_bar}", "elapsed: {elapsed}", diff --git a/src/fsentry.rs b/src/fsentry.rs index aae1475..fd09ea2 100644 --- a/src/fsentry.rs +++ b/src/fsentry.rs @@ -57,6 +57,30 @@ impl FilesystemEntry { pub fn len(&self) -> u64 { self.len } + + pub fn mode(&self) -> u32 { + self.mode + } + + pub fn atime(&self) -> i64 { + self.atime + } + + pub fn atime_ns(&self) -> i64 { + self.atime_ns + } + + pub fn mtime(&self) -> i64 { + self.mtime + } + + pub fn mtime_ns(&self) -> i64 { + self.mtime_ns + } + + pub fn is_dir(&self) -> bool { + self.kind() == FilesystemKind::Directory + } } /// Different types of file system entries. -- cgit v1.2.1