1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
use crate::fsentry::{FilesystemEntry, FsEntryError};
use log::{debug, warn};
use std::path::{Path, PathBuf};
use walkdir::{DirEntry, IntoIter, WalkDir};
/// Filesystem entry along with additional info about it.
pub struct AnnotatedFsEntry {
pub inner: FilesystemEntry,
/// Is `entry` a valid CACHEDIR.TAG?
pub is_cachedir_tag: bool,
}
/// Iterator over file system entries in a directory tree.
pub struct FsIterator {
iter: SkipCachedirs,
}
#[derive(Debug, thiserror::Error)]
pub enum FsIterError {
#[error("walkdir failed: {0}")]
WalkDir(walkdir::Error),
#[error("failed to get file system metadata for {0}: {1}")]
Metadata(PathBuf, std::io::Error),
#[error(transparent)]
FsEntryError(#[from] FsEntryError),
}
impl FsIterator {
pub fn new(root: &Path, exclude_cache_tag_directories: bool) -> Self {
Self {
iter: SkipCachedirs::new(
WalkDir::new(root).into_iter(),
exclude_cache_tag_directories,
),
}
}
}
impl Iterator for FsIterator {
type Item = Result<AnnotatedFsEntry, FsIterError>;
fn next(&mut self) -> Option<Self::Item> {
self.iter.next()
}
}
/// Cachedir-aware adaptor for WalkDir: it skips the contents of dirs that contain CACHEDIR.TAG,
/// but still yields entries for the dir and the tag themselves.
struct SkipCachedirs {
iter: IntoIter,
exclude_cache_tag_directories: bool,
// This is the last tag we've found. `next()` will yield it before asking `iter` for more
// entries.
cachedir_tag: Option<Result<AnnotatedFsEntry, FsIterError>>,
}
impl SkipCachedirs {
fn new(iter: IntoIter, exclude_cache_tag_directories: bool) -> Self {
Self {
iter,
exclude_cache_tag_directories,
cachedir_tag: None,
}
}
fn try_enqueue_cachedir_tag(&mut self, entry: &DirEntry) {
if !self.exclude_cache_tag_directories {
return;
}
// If this entry is not a directory, it means we already processed its
// parent dir and decided that it's not cached.
if !entry.file_type().is_dir() {
return;
}
let mut tag_path = entry.path().to_owned();
tag_path.push("CACHEDIR.TAG");
// Tags are required to be regular files -- not even symlinks are allowed.
if !tag_path.is_file() {
return;
};
const CACHEDIR_TAG: &[u8] = b"Signature: 8a477f597d28d172789f06886806bc55";
let mut content = [0u8; CACHEDIR_TAG.len()];
let mut file = if let Ok(file) = std::fs::File::open(&tag_path) {
file
} else {
return;
};
use std::io::Read;
match file.read_exact(&mut content) {
Ok(_) => (),
// If we can't read the tag file, proceed as if's not there
Err(_) => return,
}
if content == CACHEDIR_TAG {
self.iter.skip_current_dir();
self.cachedir_tag = Some(new_entry(&tag_path, true));
}
}
}
impl Iterator for SkipCachedirs {
type Item = Result<AnnotatedFsEntry, FsIterError>;
fn next(&mut self) -> Option<Self::Item> {
self.cachedir_tag.take().or_else(|| {
let next = self.iter.next();
debug!("walkdir found: {:?}", next);
match next {
None => None,
Some(Err(err)) => Some(Err(FsIterError::WalkDir(err))),
Some(Ok(entry)) => {
self.try_enqueue_cachedir_tag(&entry);
Some(new_entry(entry.path(), false))
}
}
})
}
}
fn new_entry(path: &Path, is_cachedir_tag: bool) -> Result<AnnotatedFsEntry, FsIterError> {
let meta = std::fs::symlink_metadata(path);
debug!("metadata for {:?}: {:?}", path, meta);
let meta = match meta {
Ok(meta) => meta,
Err(err) => {
warn!("failed to get metadata for {}: {}", path.display(), err);
return Err(FsIterError::Metadata(path.to_path_buf(), err));
}
};
let entry = FilesystemEntry::from_metadata(path, &meta)?;
debug!("FileSystemEntry for {:?}: {:?}", path, entry);
let annotated = AnnotatedFsEntry {
inner: entry,
is_cachedir_tag,
};
Ok(annotated)
}
|
