From 146a5fe13cba442afca954733bafd854f4448928 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 12 Dec 2020 10:44:50 +0200 Subject: feat: back up and restore symlinks --- obnam.md | 18 ++++++++++++++++++ src/client.rs | 2 ++ src/cmd/backup.rs | 12 +++++++++--- src/cmd/restore.rs | 17 +++++++++++++++++ src/fsentry.rs | 25 ++++++++++++++++++++++--- src/fsiter.rs | 8 ++++++-- subplot/data.py | 4 ++++ subplot/data.yaml | 3 +++ 8 files changed, 81 insertions(+), 8 deletions(-) diff --git a/obnam.md b/obnam.md index 9b98dae..5e86da2 100644 --- a/obnam.md +++ b/obnam.md @@ -569,6 +569,24 @@ given a manifest of the directory live restored in rest in rest.yaml then files live.yaml and rest.yaml match ~~~ +### Symbolic links + +This scenario verifies that symbolic links are restored correctly. + +~~~scenario +given an installed obnam +and a running chunk server +and a client config based on metadata.yaml +and a file live/data.dat containing some random data +and symbolink link live/link that points at data.dat +and a manifest of the directory live in live.yaml +when I run obnam backup metadata.yaml +then backup generation is GEN +when I invoke obnam restore metadata.yaml rest +given a manifest of the directory live restored in rest in rest.yaml +then files live.yaml and rest.yaml match +~~~ + diff --git a/src/client.rs b/src/client.rs index 658e3ed..745169b 100644 --- a/src/client.rs +++ b/src/client.rs @@ -59,9 +59,11 @@ impl BackupClient { e: FilesystemEntry, size: usize, ) -> anyhow::Result<(FilesystemEntry, Vec)> { + debug!("entry: {:?}", e); let ids = match e.kind() { FilesystemKind::Regular => self.read_file(e.path(), size)?, FilesystemKind::Directory => vec![], + FilesystemKind::Symlink => vec![], }; Ok((e, ids)) } diff --git a/src/cmd/backup.rs b/src/cmd/backup.rs index 8f195eb..86f1b96 100644 --- a/src/cmd/backup.rs +++ b/src/cmd/backup.rs @@ -2,6 +2,7 @@ use crate::client::{BackupClient, ClientConfig}; use crate::fsiter::FsIterator; use crate::generation::Generation; use indicatif::{ProgressBar, ProgressStyle}; +use log::info; use std::path::Path; use tempfile::NamedTempFile; @@ -24,13 +25,14 @@ pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> { // The fetching is in its own block so that the file handles // get closed and data flushed to disk. let mut gen = Generation::create(&dbname)?; - let progress = create_progress_bar(GUESS_FILE_COUNT); + let progress = create_progress_bar(GUESS_FILE_COUNT, false); progress.enable_steady_tick(100); gen.insert_iter(FsIterator::new(&config.root).map(|entry| { progress.inc(1); match entry { Err(err) => Err(err), Ok(entry) => { + info!("backup: {}", entry.path().display()); progress.set_message(&format!("{}", entry.path().display())); client.upload_filesystem_entry(entry, buffer_size) } @@ -52,8 +54,12 @@ pub fn backup(config: &Path, buffer_size: usize) -> anyhow::Result<()> { Ok(()) } -fn create_progress_bar(file_count: u64) -> ProgressBar { - let progress = ProgressBar::new(file_count); +fn create_progress_bar(file_count: u64, verbose: bool) -> ProgressBar { + let progress = if verbose { + ProgressBar::new(file_count) + } else { + ProgressBar::hidden() + }; let parts = vec![ "{wide_bar}", "elapsed: {elapsed}", diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index 2b62ef5..27f0ce3 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -8,6 +8,7 @@ use log::{debug, error, info}; use std::fs::File; use std::io::prelude::*; use std::io::Error; +use std::os::unix::fs::symlink; use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use structopt::StructOpt; @@ -81,6 +82,7 @@ fn restore_generation( to: &Path, progress: &ProgressBar, ) -> anyhow::Result<()> { + debug!("restoring {:?}", entry); progress.set_message(&format!("{}", entry.path().display())); progress.inc(1); @@ -88,6 +90,7 @@ fn restore_generation( match entry.kind() { FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?, FilesystemKind::Directory => restore_directory(&to)?, + FilesystemKind::Symlink => restore_symlink(&to, &entry)?, } Ok(()) } @@ -142,6 +145,20 @@ fn restore_regular( Ok(()) } +fn restore_symlink(path: &Path, entry: &FilesystemEntry) -> anyhow::Result<()> { + debug!("restoring symlink {}", path.display()); + let parent = path.parent().unwrap(); + debug!(" mkdir {}", parent.display()); + if !parent.exists() { + std::fs::create_dir_all(parent)?; + { + symlink(path, entry.symlink_target().unwrap())?; + } + } + debug!("restored regular {}", path.display()); + Ok(()) +} + fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> anyhow::Result<()> { debug!("restoring metadata for {}", entry.path().display()); diff --git a/src/fsentry.rs b/src/fsentry.rs index fd09ea2..c66e3e0 100644 --- a/src/fsentry.rs +++ b/src/fsentry.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::fs::read_link; use std::fs::{FileType, Metadata}; use std::os::linux::fs::MetadataExt; use std::path::{Path, PathBuf}; @@ -29,12 +30,16 @@ pub struct FilesystemEntry { mtime_ns: i64, atime: i64, atime_ns: i64, + + // The target of a symbolic link, if any. + symlink_target: Option, } #[allow(clippy::len_without_is_empty)] impl FilesystemEntry { - pub fn from_metadata(path: &Path, meta: &Metadata) -> Self { - Self { + pub fn from_metadata(path: &Path, meta: &Metadata) -> anyhow::Result { + let kind = FilesystemKind::from_file_type(meta.file_type()); + Ok(Self { path: path.to_path_buf(), kind: FilesystemKind::from_file_type(meta.file_type()), len: meta.len(), @@ -43,7 +48,12 @@ impl FilesystemEntry { mtime_ns: meta.st_mtime_nsec(), atime: meta.st_atime(), atime_ns: meta.st_atime_nsec(), - } + symlink_target: if kind == FilesystemKind::Symlink { + Some(read_link(path)?) + } else { + None + }, + }) } pub fn kind(&self) -> FilesystemKind { @@ -81,6 +91,10 @@ impl FilesystemEntry { pub fn is_dir(&self) -> bool { self.kind() == FilesystemKind::Directory } + + pub fn symlink_target(&self) -> Option { + self.symlink_target.clone() + } } /// Different types of file system entries. @@ -88,6 +102,7 @@ impl FilesystemEntry { pub enum FilesystemKind { Regular, Directory, + Symlink, } impl FilesystemKind { @@ -96,6 +111,8 @@ impl FilesystemKind { FilesystemKind::Regular } else if file_type.is_dir() { FilesystemKind::Directory + } else if file_type.is_symlink() { + FilesystemKind::Symlink } else { panic!("unknown file type {:?}", file_type); } @@ -105,6 +122,7 @@ impl FilesystemKind { match self { FilesystemKind::Regular => 0, FilesystemKind::Directory => 1, + FilesystemKind::Symlink => 2, } } @@ -112,6 +130,7 @@ impl FilesystemKind { match code { 0 => Ok(FilesystemKind::Regular), 1 => Ok(FilesystemKind::Directory), + 2 => Ok(FilesystemKind::Symlink), _ => Err(Error::UnknownFileKindCode(code).into()), } } diff --git a/src/fsiter.rs b/src/fsiter.rs index 3c08179..a40ad34 100644 --- a/src/fsiter.rs +++ b/src/fsiter.rs @@ -1,4 +1,5 @@ use crate::fsentry::FilesystemEntry; +use log::info; use std::path::Path; use walkdir::{IntoIter, WalkDir}; @@ -20,7 +21,10 @@ impl Iterator for FsIterator { fn next(&mut self) -> Option { match self.iter.next() { None => None, - Some(Ok(entry)) => Some(new_entry(&entry)), + Some(Ok(entry)) => { + info!("found {}", entry.path().display()); + Some(new_entry(&entry)) + } Some(Err(err)) => Some(Err(err.into())), } } @@ -28,6 +32,6 @@ impl Iterator for FsIterator { fn new_entry(e: &walkdir::DirEntry) -> anyhow::Result { let meta = e.metadata()?; - let entry = FilesystemEntry::from_metadata(e.path(), &meta); + let entry = FilesystemEntry::from_metadata(e.path(), &meta)?; Ok(entry) } diff --git a/subplot/data.py b/subplot/data.py index 09df6c5..ba3636c 100644 --- a/subplot/data.py +++ b/subplot/data.py @@ -17,6 +17,10 @@ def chmod_file(ctx, filename=None, mode=None): os.chmod(filename, int(mode, 8)) +def create_symlink(ctx, linkname=None, target=None): + os.symlink(linkname, target) + + def create_manifest_of_live(ctx, dirname=None, manifest=None): _create_manifest_of_directory(ctx, dirname=dirname, manifest=manifest) diff --git a/subplot/data.yaml b/subplot/data.yaml index 3b4d938..32c9cd5 100644 --- a/subplot/data.yaml +++ b/subplot/data.yaml @@ -9,6 +9,9 @@ - given: file {filename} has mode {mode} function: chmod_file +- given: symbolink link {linkname} that points at {target} + function: create_symlink + - given: a manifest of the directory {dirname} in {manifest} function: create_manifest_of_live -- cgit v1.2.1