feat: back up and restore Unix domain sockets

11 files changed, 118 insertions, 34 deletions

diff --git a/client.yaml b/client.yaml
index 4386b90..aead475 100644
--- a/client.yaml
+++ b/client.yaml
@@ -1,5 +1,5 @@
 server_url: https://localhost:8888
 verify_tls_cert: false
 roots:
-  - /home/liw/pers/debian-ansible
+  - /home/liw/tmp/irregular
 log: obnam.log
diff --git a/obnam.md b/obnam.md
index cad3792..e386b17 100644
--- a/obnam.md
+++ b/obnam.md
@@ -1283,6 +1283,24 @@ when I invoke obnam --config smoke.yaml get-chunk <GEN>
 then command fails
 ~~~
 
+## Irregular files
+
+This scenario verifies that Obnam backs up and restores files that
+aren't regular files or directories.
+
+~~~scenario
+given an installed obnam
+and a running chunk server
+and a client config based on smoke.yaml
+and a file live/data.dat containing some random data
+and a Unix socket live/socket
+and a manifest of the directory live in live.yaml
+when I run obnam --config smoke.yaml backup
+when I invoke obnam --config smoke.yaml restore latest rest
+given a manifest of the directory live restored in rest in rest.yaml
+then files live.yaml and rest.yaml match
+~~~
+
 ## Tricky filenames
 
 Obnam needs to handle all filenames the underlying operating and file
diff --git a/src/backup_run.rs b/src/backup_run.rs
index 024f486..60623e6 100644
--- a/src/backup_run.rs
+++ b/src/backup_run.rs
@@ -55,7 +55,11 @@ impl<'a> InitialBackup<'a> {
         entry: FsIterResult<FilesystemEntry>,
     ) -> BackupResult<(FilesystemEntry, Vec<ChunkId>, Reason)> {
         match entry {
-            Err(err) => Err(err.into()),
+            Err(err) => {
+                warn!("backup: there was a problem: {:?}", err);
+                self.progress.found_problem();
+                Err(err.into())
+            }
             Ok(entry) => {
                 let path = &entry.pathbuf();
                 info!("backup: {}", path.display());
diff --git a/src/bin/obnam.rs b/src/bin/obnam.rs
index 8778a73..c163695 100644
--- a/src/bin/obnam.rs
+++ b/src/bin/obnam.rs
@@ -29,7 +29,7 @@ fn main() -> anyhow::Result<()> {
     };
 
     if let Err(ref e) = result {
-        error!("{}", e);
+        error!("command failed: {}", e);
         eprintln!("ERROR: {}", e);
         result?
     }
diff --git a/src/client.rs b/src/client.rs
index a475ce9..9d1e8ff 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -157,18 +157,21 @@ impl BackupClient {
         e: &FilesystemEntry,
         size: usize,
     ) -> ClientResult<Vec<ChunkId>> {
-        info!("upload entry: {:?}", e);
+        let path = e.pathbuf();
+        info!("uploading {:?}", path);
         let ids = match e.kind() {
-            FilesystemKind::Regular => self.read_file(e.pathbuf(), size)?,
+            FilesystemKind::Regular => self.read_file(&path, size)?,
             FilesystemKind::Directory => vec![],
             FilesystemKind::Symlink => vec![],
+            FilesystemKind::Socket => vec![],
         };
+        info!("upload OK for {:?}", path);
         Ok(ids)
     }
 
     pub fn upload_generation(&self, filename: &Path, size: usize) -> ClientResult<ChunkId> {
         info!("upload SQLite {}", filename.display());
-        let ids = self.read_file(filename.to_path_buf(), size)?;
+        let ids = self.read_file(filename, size)?;
         let gen = GenerationChunk::new(ids);
         let data = gen.to_data_chunk()?;
         let meta = ChunkMeta::new_generation(&sha256(data.data()), &current_timestamp());
@@ -177,7 +180,7 @@ impl BackupClient {
         Ok(gen_id)
     }
 
-    fn read_file(&self, filename: PathBuf, size: usize) -> ClientResult<Vec<ChunkId>> {
+    fn read_file(&self, filename: &Path, size: usize) -> ClientResult<Vec<ChunkId>> {
         info!("upload file {}", filename.display());
         let file = std::fs::File::open(filename)?;
         let chunker = Chunker::new(size, file);
diff --git a/src/cmd/list_files.rs b/src/cmd/list_files.rs
index 38048ec..4982cc2 100644
--- a/src/cmd/list_files.rs
+++ b/src/cmd/list_files.rs
@@ -35,6 +35,7 @@ fn format_entry(e: &FilesystemEntry, reason: Reason) -> String {
         FilesystemKind::Regular => "-",
         FilesystemKind::Directory => "d",
         FilesystemKind::Symlink => "l",
+        FilesystemKind::Socket => "s",
     };
     format!("{} {} ({})", kind, e.pathbuf().display(), reason)
 }
diff --git a/src/cmd/restore.rs b/src/cmd/restore.rs
index 5d01bd4..147a56b 100644
--- a/src/cmd/restore.rs
+++ b/src/cmd/restore.rs
@@ -5,13 +5,14 @@ use crate::error::ObnamError;
 use crate::fsentry::{FilesystemEntry, FilesystemKind};
 use crate::generation::{LocalGeneration, LocalGenerationError};
 use indicatif::{ProgressBar, ProgressStyle};
-use libc::{fchmod, futimens, timespec};
+use libc::{chmod, timespec, utimensat, AT_FDCWD};
 use log::{debug, error, info};
-use std::fs::File;
+use std::ffi::CString;
 use std::io::prelude::*;
 use std::io::Error;
+use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::symlink;
-use std::os::unix::io::AsRawFd;
+use std::os::unix::net::UnixListener;
 use std::path::StripPrefixError;
 use std::path::{Path, PathBuf};
 use structopt::StructOpt;
@@ -86,6 +87,9 @@ pub enum RestoreError {
 
     #[error(transparent)]
     SerdeYamlError(#[from] serde_yaml::Error),
+
+    #[error(transparent)]
+    NulError(#[from] std::ffi::NulError),
 }
 
 pub type RestoreResult<T> = Result<T, RestoreError>;
@@ -107,6 +111,7 @@ fn restore_generation(
         FilesystemKind::Regular => restore_regular(client, &gen, &to, fileid, &entry)?,
         FilesystemKind::Directory => restore_directory(&to)?,
         FilesystemKind::Symlink => restore_symlink(&to, &entry)?,
+        FilesystemKind::Socket => restore_socket(&to, &entry)?,
     }
     Ok(())
 }
@@ -176,10 +181,17 @@ fn restore_symlink(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> {
     Ok(())
 }
 
+fn restore_socket(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> {
+    debug!("creating Unix domain socket {:?}", path);
+    UnixListener::bind(path)?;
+    restore_metadata(path, entry)?;
+    Ok(())
+}
+
 fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> {
     debug!("restoring metadata for {}", entry.pathbuf().display());
 
-    let handle = File::open(path)?;
+    debug!("restoring metadata for {:?}", path);
 
     let atime = timespec {
         tv_sec: entry.atime(),
@@ -192,28 +204,35 @@ fn restore_metadata(path: &Path, entry: &FilesystemEntry) -> RestoreResult<()> {
     let times = [atime, mtime];
     let times: *const timespec = &times[0];
 
+    let path = path_to_cstring(path);
+
     // We have to use unsafe here to be able call the libc functions
     // below.
     unsafe {
-        let fd = handle.as_raw_fd(); // FIXME: needs to NOT follow symlinks
-
-        debug!("fchmod");
-        if fchmod(fd, entry.mode()) == -1 {
+        debug!("chmod {:?}", path);
+        if chmod(path.as_ptr(), entry.mode()) == -1 {
             let error = Error::last_os_error();
-            error!("fchmod failed on {}", path.display());
+            error!("chmod failed on {:?}", path);
             return Err(error.into());
         }
 
-        debug!("futimens");
-        if futimens(fd, times) == -1 {
+        debug!("utimens {:?}", path);
+        if utimensat(AT_FDCWD, path.as_ptr(), times, 0) == -1 {
             let error = Error::last_os_error();
-            error!("futimens failed on {}", path.display());
+            error!("utimensat failed on {:?}", path);
             return Err(error.into());
         }
     }
     Ok(())
 }
 
+fn path_to_cstring(path: &Path) -> CString {
+    let path = path.as_os_str();
+    let path = path.as_bytes();
+    let path = CString::new(path).unwrap();
+    path
+}
+
 fn create_progress_bar(file_count: i64, verbose: bool) -> ProgressBar {
     let progress = if verbose {
         ProgressBar::new(file_count as u64)
diff --git a/src/fsentry.rs b/src/fsentry.rs
index 89c1cc0..9384ec6 100644
--- a/src/fsentry.rs
+++ b/src/fsentry.rs
@@ -1,9 +1,11 @@
+use log::{debug, error};
 use serde::{Deserialize, Serialize};
 use std::ffi::OsString;
 use std::fs::read_link;
 use std::fs::{FileType, Metadata};
 use std::os::linux::fs::MetadataExt;
 use std::os::unix::ffi::OsStringExt;
+use std::os::unix::fs::FileTypeExt;
 use std::path::{Path, PathBuf};
 
 /// A file system entry.
@@ -52,6 +54,19 @@ pub type FsEntryResult<T> = Result<T, FsEntryError>;
 impl FilesystemEntry {
     pub fn from_metadata(path: &Path, meta: &Metadata) -> FsEntryResult<Self> {
         let kind = FilesystemKind::from_file_type(meta.file_type());
+        let symlink_target = if kind == FilesystemKind::Symlink {
+            debug!("reading symlink target for {:?}", path);
+            let target = match read_link(path) {
+                Ok(x) => x,
+                Err(err) => {
+                    error!("read_link failed: {}", err);
+                    return Err(err.into());
+                }
+            };
+            Some(target)
+        } else {
+            None
+        };
         Ok(Self {
             path: path.to_path_buf().into_os_string().into_vec(),
             kind: FilesystemKind::from_file_type(meta.file_type()),
@@ -61,11 +76,7 @@ impl FilesystemEntry {
             mtime_ns: meta.st_mtime_nsec(),
             atime: meta.st_atime(),
             atime_ns: meta.st_atime_nsec(),
-            symlink_target: if kind == FilesystemKind::Symlink {
-                Some(read_link(path)?)
-            } else {
-                None
-            },
+            symlink_target,
         })
     }
 
@@ -117,6 +128,7 @@ pub enum FilesystemKind {
     Regular,
     Directory,
     Symlink,
+    Socket,
 }
 
 impl FilesystemKind {
@@ -127,6 +139,8 @@ impl FilesystemKind {
             FilesystemKind::Directory
         } else if file_type.is_symlink() {
             FilesystemKind::Symlink
+        } else if file_type.is_socket() {
+            FilesystemKind::Socket
         } else {
             panic!("unknown file type {:?}", file_type);
         }
@@ -137,6 +151,7 @@ impl FilesystemKind {
             FilesystemKind::Regular => 0,
             FilesystemKind::Directory => 1,
             FilesystemKind::Symlink => 2,
+            FilesystemKind::Socket => 3,
         }
     }
 
@@ -145,6 +160,7 @@ impl FilesystemKind {
             0 => Ok(FilesystemKind::Regular),
             1 => Ok(FilesystemKind::Directory),
             2 => Ok(FilesystemKind::Symlink),
+            3 => Ok(FilesystemKind::Socket),
             _ => Err(FsEntryError::UnknownFileKindCode(code).into()),
         }
     }
@@ -164,6 +180,8 @@ mod test {
     fn file_kind_regular_round_trips() {
         one_file_kind_round_trip(FilesystemKind::Regular);
         one_file_kind_round_trip(FilesystemKind::Directory);
+        one_file_kind_round_trip(FilesystemKind::Symlink);
+        one_file_kind_round_trip(FilesystemKind::Socket);
     }
 
     fn one_file_kind_round_trip(kind: FilesystemKind) {
diff --git a/src/fsiter.rs b/src/fsiter.rs
index 36693a6..f59fb64 100644
--- a/src/fsiter.rs
+++ b/src/fsiter.rs
@@ -1,7 +1,7 @@
 use crate::fsentry::{FilesystemEntry, FsEntryError};
-use log::info;
+use log::{debug, error};
 use std::path::Path;
-use walkdir::{IntoIter, WalkDir};
+use walkdir::{DirEntry, IntoIter, WalkDir};
 
 /// Iterator over file system entries in a directory tree.
 pub struct FsIterator {
@@ -14,6 +14,9 @@ pub enum FsIterError {
     WalkError(#[from] walkdir::Error),
 
     #[error(transparent)]
+    IoError(#[from] std::io::Error),
+
+    #[error(transparent)]
     FsEntryError(#[from] FsEntryError),
 }
 
@@ -30,19 +33,28 @@ impl FsIterator {
 impl Iterator for FsIterator {
     type Item = FsIterResult<FilesystemEntry>;
     fn next(&mut self) -> Option<Self::Item> {
-        match self.iter.next() {
+        let next = self.iter.next();
+        debug!("walkdir found: {:?}", next);
+        match next {
             None => None,
-            Some(Ok(entry)) => {
-                info!("found {}", entry.path().display());
-                Some(new_entry(&entry))
-            }
+            Some(Ok(entry)) => Some(new_entry(&entry)),
             Some(Err(err)) => Some(Err(err.into())),
         }
     }
 }
 
-fn new_entry(e: &walkdir::DirEntry) -> FsIterResult<FilesystemEntry> {
-    let meta = e.metadata()?;
-    let entry = FilesystemEntry::from_metadata(e.path(), &meta)?;
+fn new_entry(e: &DirEntry) -> FsIterResult<FilesystemEntry> {
+    let path = e.path();
+    let meta = std::fs::metadata(path);
+    debug!("metadata for {:?}: {:?}", path, meta);
+    let meta = match meta {
+        Ok(meta) => meta,
+        Err(err) => {
+            error!("failed to get metadata: {}", err);
+            return Err(err.into());
+        }
+    };
+    let entry = FilesystemEntry::from_metadata(path, &meta)?;
+    debug!("FileSystemEntry for {:?}: {:?}", path, entry);
     Ok(entry)
 }
diff --git a/subplot/data.py b/subplot/data.py
index f7fb903..3c369d1 100644
--- a/subplot/data.py
+++ b/subplot/data.py
@@ -2,6 +2,7 @@ import json
 import logging
 import os
 import random
+import socket
 import yaml
 
 
@@ -18,6 +19,11 @@ def create_file_with_random_data(ctx, filename=None):
     create_file_with_given_data(ctx, filename=filename, data=data)
 
 
+def create_unix_socket(ctx, filename=None):
+    fd = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    fd.bind(filename)
+
+
 def create_nonutf8_filename(ctx, dirname=None):
     filename = "\x88"
     os.mkdir(dirname)
diff --git a/subplot/data.yaml b/subplot/data.yaml
index 1636e77..97e6eed 100644
--- a/subplot/data.yaml
+++ b/subplot/data.yaml
@@ -4,6 +4,9 @@
 - given: "a file {filename} containing some random data"
   function: create_file_with_random_data
 
+- given: "a Unix socket {filename}"
+  function: create_unix_socket
+
 - given: "a file in {dirname} with a non-UTF8 filename"
   function: create_nonutf8_filename