From 248e5ab7518746c0ac43747040290e9b5d138028 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 18 Feb 2021 09:27:25 +0200 Subject: feat: back up and restore Unix domain sockets --- client.yaml | 2 +- obnam.md | 18 ++++++++++++++++++ src/backup_run.rs | 6 +++++- src/bin/obnam.rs | 2 +- src/client.rs | 11 +++++++---- src/cmd/list_files.rs | 1 + src/cmd/restore.rs | 43 +++++++++++++++++++++++++++++++------------ src/fsentry.rs | 28 +++++++++++++++++++++++----- src/fsiter.rs | 32 ++++++++++++++++++++++---------- subplot/data.py | 6 ++++++ subplot/data.yaml | 3 +++ 11 files changed, 118 insertions(+), 34 deletions(-) diff --git a/client.yaml b/client.yaml index 4386b90..aead475 100644 --- a/client.yaml +++ b/client.yaml @@ -1,5 +1,5 @@ server_url: https://localhost:8888 verify_tls_cert: false roots: - - /home/liw/pers/debian-ansible + - /home/liw/tmp/irregular log: obnam.log diff --git a/obnam.md b/obnam.md index cad3792..e386b17 100644 --- a/obnam.md +++ b/obnam.md @@ -1283,6 +1283,24 @@ when I invoke obnam --config smoke.yaml get-chunk then command fails ~~~ +## Irregular files + +This scenario verifies that Obnam backs up and restores files that +aren't regular files or directories. + +~~~scenario +given an installed obnam +and a running chunk server +and a client config based on smoke.yaml +and a file live/data.dat containing some random data +and a Unix socket live/socket +and a manifest of the directory live in live.yaml +when I run obnam --config smoke.yaml backup +when I invoke obnam --config smoke.yaml restore latest rest +given a manifest of the directory live restored in rest in rest.yaml +then files live.yaml and rest.yaml match +~~~ + ## Tricky filenames Obnam needs to handle all filenames the underlying operating and file diff --git a/src/backup_run.rs b/src/backup_run.rs index 024f486..60623e6 100644 --- a/src/backup_run.rs +++ b/src/backup_run.rs @@ -55,7 +55,11 @@ impl<'a> InitialBackup<'a> { entry: FsIterResult, ) -> BackupResult<(FilesystemEntry, Vec, Reason)> { match entry { - Err(err) => Err(err.into()), + Err(err) => { + warn!("backup: there was a problem: {:?}", err); + self.progress.found_problem(); + Err(err.into()) + } Ok(entry) => { let path = &entry.pathbuf(); info!("backup: {}", path.display()); diff --git a/src/bin/obnam.rs b/src/bin/obnam.rs index 8778a73..c163695 100644 --- a/src/bin/obnam.rs +++ b/src/bin/obnam.rs @@ -29,7 +29,7 @@ fn main() -> anyhow::Result<()> { }; if let Err(ref e) = result { - error!("{}", e); + error!("command failed: {}", e); eprintln!("ERROR: {}", e); result? } diff --git a/src/client.rs b/src/client.rs index a475ce9..9d1e8ff 100644 --- a/src/client.rs +++ b/src/client.rs @@ -157,18 +157,21 @@ impl BackupClient { e: &FilesystemEntry, size: usize, ) -> ClientResult> { - info!("upload entry: {:?}", e); + let path = e.pathbuf(); + info!("uploading {:?}", path); let ids = match e.kind() { - FilesystemKind::Regular => self.read_file(e.pathbuf(), size)?, + FilesystemKind::Regular => self.read_file(&path, size)?, FilesystemKind::Directory => vec![], FilesystemKind::Symlink => vec![], + FilesystemKind::Socket => vec![], }; + info!("upload OK for {:?}", path); Ok(ids) } pub fn upload_generation(&self, filename: &Path, size: usize) -> ClientResult { info!("upload SQLite {}", filename.display()); - let ids = self.read_file(filename.to_path_buf(), size)?; + let ids = self.read_file(filename, size)?; let gen = GenerationChunk::new(ids); let data = gen.to_data_chunk()?; let meta = ChunkMeta::new_generation(&sha256(data.data()), ¤t_timestamp()); @@ -177,7 +180,7 @@ impl BackupClient { Ok(gen_id) } - fn read_file(&self, filename: PathBuf, size: usize) -> ClientResult> { + fn read_file(&self, filename: &Path, size: usize) -> ClientResult> { info!("upload file {}", filename.display()); let file = std::fs::File::open(filename)?; let chunker = Chunker::new(size, file); diff --git a/src/cmd/list_files.rs b/src/cmd/list_files.rs index 38048ec..4982cc2 100644 --- a/src/cmd/list_files.rs +++ b/src/cmd/list_files.rs @@ -35,6 +35,7 @@ fn format_entry(e: &FilesystemEntry, reason: Reason) -> String { FilesystemKind::Regular => "-", FilesystemKind::Directory => "d", FilesystemKind::Symlink => "l", + FilesystemKind::Socket => "s", }; format!("{} {} ({})", kind, e.pathbuf().display(), reason) } diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs index 5d01bd4..147a56b 100644 --- a/src/cmd/restore.rs +++ b/src/cmd/restore.rs @@ -5,13 +5,14 @@ use crate::error::ObnamError; use crate::fsentry::{FilesystemEntry, FilesystemKind}; use crate::generation::{LocalGeneration, LocalGenerationError}; use indicatif::{ProgressBar, ProgressStyle}; -use libc::{fchmod, futimens, timespec}; +use libc::{chmod, timespec, utimensat, AT_FDCWD}; use log::{debug, error, info}; -use std::fs::File; +use std::ffi::CString; use std::io::prelude::*; use std::io::Error; +use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::symlink; -use std::os::unix::io::AsRawFd; +use std::os::unix::net::UnixListener; use std::path::StripPrefixError; use std::path::{Path, PathBuf}; use structopt::StructOpt; @@ -86,6 +87,9 @@ pub enum RestoreError { #[error(transparent)] SerdeYamlError(#[from] serde_yaml::Error), + + #[error(transparent)] + NulError(#[from] std::ffi::NulError), } pub type RestoreResult = Result; @@ -107,6 +111,7 @@ fn restore_generation( FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?, FilesystemKind::Directory => restore_directory(&to)?, FilesystemKind::Symlink => restore_symlink(&to, &entry)?, + FilesystemKind::Socket => restore_socket(&to, &entry)?, } Ok(()) } @@ -176,10 +181,17 @@ fn restore_symlink(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> { Ok(()) } +fn restore_socket(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> { + debug!("creating Unix domain socket {:?}", path); + UnixListener::bind(path)?; + restore_metadata(path, entry)?; + Ok(()) +} + fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> { debug!("restoring metadata for {}", entry.pathbuf().display()); - let handle = File::open(path)?; + debug!("restoring metadata for {:?}", path); let atime = timespec { tv_sec: entry.atime(), @@ -192,28 +204,35 @@ fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> { let times = [atime, mtime]; let times: *const timespec = ×[0]; + let path = path_to_cstring(path); + // We have to use unsafe here to be able call the libc functions // below. unsafe { - let fd = handle.as_raw_fd(); // FIXME: needs to NOT follow symlinks - - debug!("fchmod"); - if fchmod(fd, entry.mode()) == -1 { + debug!("chmod {:?}", path); + if chmod(path.as_ptr(), entry.mode()) == -1 { let error = Error::last_os_error(); - error!("fchmod failed on {}", path.display()); + error!("chmod failed on {:?}", path); return Err(error.into()); } - debug!("futimens"); - if futimens(fd, times) == -1 { + debug!("utimens {:?}", path); + if utimensat(AT_FDCWD, path.as_ptr(), times, 0) == -1 { let error = Error::last_os_error(); - error!("futimens failed on {}", path.display()); + error!("utimensat failed on {:?}", path); return Err(error.into()); } } Ok(()) } +fn path_to_cstring(path: &Path) -> CString { + let path = path.as_os_str(); + let path = path.as_bytes(); + let path = CString::new(path).unwrap(); + path +} + fn create_progress_bar(file_count: i64, verbose: bool) -> ProgressBar { let progress = if verbose { ProgressBar::new(file_count as u64) diff --git a/src/fsentry.rs b/src/fsentry.rs index 89c1cc0..9384ec6 100644 --- a/src/fsentry.rs +++ b/src/fsentry.rs @@ -1,9 +1,11 @@ +use log::{debug, error}; use serde::{Deserialize, Serialize}; use std::ffi::OsString; use std::fs::read_link; use std::fs::{FileType, Metadata}; use std::os::linux::fs::MetadataExt; use std::os::unix::ffi::OsStringExt; +use std::os::unix::fs::FileTypeExt; use std::path::{Path, PathBuf}; /// A file system entry. @@ -52,6 +54,19 @@ pub type FsEntryResult = Result; impl FilesystemEntry { pub fn from_metadata(path: &Path, meta: &Metadata) -> FsEntryResult { let kind = FilesystemKind::from_file_type(meta.file_type()); + let symlink_target = if kind == FilesystemKind::Symlink { + debug!("reading symlink target for {:?}", path); + let target = match read_link(path) { + Ok(x) => x, + Err(err) => { + error!("read_link failed: {}", err); + return Err(err.into()); + } + }; + Some(target) + } else { + None + }; Ok(Self { path: path.to_path_buf().into_os_string().into_vec(), kind: FilesystemKind::from_file_type(meta.file_type()), @@ -61,11 +76,7 @@ impl FilesystemEntry { mtime_ns: meta.st_mtime_nsec(), atime: meta.st_atime(), atime_ns: meta.st_atime_nsec(), - symlink_target: if kind == FilesystemKind::Symlink { - Some(read_link(path)?) - } else { - None - }, + symlink_target, }) } @@ -117,6 +128,7 @@ pub enum FilesystemKind { Regular, Directory, Symlink, + Socket, } impl FilesystemKind { @@ -127,6 +139,8 @@ impl FilesystemKind { FilesystemKind::Directory } else if file_type.is_symlink() { FilesystemKind::Symlink + } else if file_type.is_socket() { + FilesystemKind::Socket } else { panic!("unknown file type {:?}", file_type); } @@ -137,6 +151,7 @@ impl FilesystemKind { FilesystemKind::Regular => 0, FilesystemKind::Directory => 1, FilesystemKind::Symlink => 2, + FilesystemKind::Socket => 3, } } @@ -145,6 +160,7 @@ impl FilesystemKind { 0 => Ok(FilesystemKind::Regular), 1 => Ok(FilesystemKind::Directory), 2 => Ok(FilesystemKind::Symlink), + 3 => Ok(FilesystemKind::Socket), _ => Err(FsEntryError::UnknownFileKindCode(code).into()), } } @@ -164,6 +180,8 @@ mod test { fn file_kind_regular_round_trips() { one_file_kind_round_trip(FilesystemKind::Regular); one_file_kind_round_trip(FilesystemKind::Directory); + one_file_kind_round_trip(FilesystemKind::Symlink); + one_file_kind_round_trip(FilesystemKind::Socket); } fn one_file_kind_round_trip(kind: FilesystemKind) { diff --git a/src/fsiter.rs b/src/fsiter.rs index 36693a6..f59fb64 100644 --- a/src/fsiter.rs +++ b/src/fsiter.rs @@ -1,7 +1,7 @@ use crate::fsentry::{FilesystemEntry, FsEntryError}; -use log::info; +use log::{debug, error}; use std::path::Path; -use walkdir::{IntoIter, WalkDir}; +use walkdir::{DirEntry, IntoIter, WalkDir}; /// Iterator over file system entries in a directory tree. pub struct FsIterator { @@ -13,6 +13,9 @@ pub enum FsIterError { #[error(transparent)] WalkError(#[from] walkdir::Error), + #[error(transparent)] + IoError(#[from] std::io::Error), + #[error(transparent)] FsEntryError(#[from] FsEntryError), } @@ -30,19 +33,28 @@ impl FsIterator { impl Iterator for FsIterator { type Item = FsIterResult; fn next(&mut self) -> Option { - match self.iter.next() { + let next = self.iter.next(); + debug!("walkdir found: {:?}", next); + match next { None => None, - Some(Ok(entry)) => { - info!("found {}", entry.path().display()); - Some(new_entry(&entry)) - } + Some(Ok(entry)) => Some(new_entry(&entry)), Some(Err(err)) => Some(Err(err.into())), } } } -fn new_entry(e: &walkdir::DirEntry) -> FsIterResult { - let meta = e.metadata()?; - let entry = FilesystemEntry::from_metadata(e.path(), &meta)?; +fn new_entry(e: &DirEntry) -> FsIterResult { + let path = e.path(); + let meta = std::fs::metadata(path); + debug!("metadata for {:?}: {:?}", path, meta); + let meta = match meta { + Ok(meta) => meta, + Err(err) => { + error!("failed to get metadata: {}", err); + return Err(err.into()); + } + }; + let entry = FilesystemEntry::from_metadata(path, &meta)?; + debug!("FileSystemEntry for {:?}: {:?}", path, entry); Ok(entry) } diff --git a/subplot/data.py b/subplot/data.py index f7fb903..3c369d1 100644 --- a/subplot/data.py +++ b/subplot/data.py @@ -2,6 +2,7 @@ import json import logging import os import random +import socket import yaml @@ -18,6 +19,11 @@ def create_file_with_random_data(ctx, filename=None): create_file_with_given_data(ctx, filename=filename, data=data) +def create_unix_socket(ctx, filename=None): + fd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + fd.bind(filename) + + def create_nonutf8_filename(ctx, dirname=None): filename = "\x88" os.mkdir(dirname) diff --git a/subplot/data.yaml b/subplot/data.yaml index 1636e77..97e6eed 100644 --- a/subplot/data.yaml +++ b/subplot/data.yaml @@ -4,6 +4,9 @@ - given: "a file {filename} containing some random data" function: create_file_with_random_data +- given: "a Unix socket {filename}" + function: create_unix_socket + - given: "a file in {dirname} with a non-UTF8 filename" function: create_nonutf8_filename -- cgit v1.2.1