1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
|
# Copyright (C) 2008 Lars Wirzenius <liw@iki.fi>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""Main program for Obnam."""
import logging
import os
import re
import stat
import time
import obnam
# Maximum number of files per file group we create.
MAX_PER_FILEGROUP = 16
class Application:
"""Main program logic for Obnam, a backup application."""
def __init__(self, context):
self._context = context
self._exclusion_strings = []
self._exclusion_regexps = []
self._filelist = None
self._store = obnam.Store(self._context)
# When we traverse the file system tree while making a backup,
# we process children before the parent. This is necessary for
# functional updates of trees. For every directory, we need
# to keep track of its children. This dict is used for that.
# It is indexed by the absolute path to the directory, and
# contains a list of the subdirectories in that directory.
# When we're done with a directory (i.e., we generate its
# DirObject), we remove the directory from this dict. This
# means that we need only data for one path from the root of
# the directory tree to the current directory, not for the
# entire directory tree.
self._subdirs = {}
def get_context(self):
"""Get the context for the backup application."""
return self._context
def get_store(self):
"""Get the Store for the backup application."""
return self._store
def load_host(self):
"""Load the host block into memory."""
self.get_store().fetch_host_block()
return self.get_store().get_host_block()
def get_exclusion_regexps(self):
"""Return list of regexp to exclude things from backup."""
config = self.get_context().config
strings = config.getvalues("backup", "exclude")
if self._exclusion_strings != strings:
for string in strings:
logging.debug("Compiling exclusion pattern '%s'" % string)
self._exclusion_regexps.append(re.compile(string))
return self._exclusion_regexps
def prune(self, dirname, dirnames, filenames):
"""Remove excluded items from dirnames and filenames.
Because this is called by obnam.walk.depth_first, the lists
are modified in place.
"""
self._prune_one_list(dirname, dirnames)
self._prune_one_list(dirname, filenames)
def _prune_one_list(self, dirname, basenames):
"""Prune one list of basenames based on exlusion list.
Because this is called from self.prune, the list is modified
in place.
"""
dirname = obnam.io.unsolve(self._context, dirname)
i = 0
while i < len(basenames):
path = os.path.join(dirname, basenames[i])
for regexp in self.get_exclusion_regexps():
if regexp.search(path):
logging.debug("Excluding %s" % path)
logging.debug(" based on %s" % regexp.pattern)
del basenames[i]
break
else:
i += 1
def set_prevgen_filelist(self, filelist):
"""Set the Filelist object from the previous generation.
This is used when looking up files in previous generations. We
only look at one generation's Filelist, since they're big. Note
that Filelist objects are the _old_ way of storing file meta
data, and we will no use better ways that let us look further
back in history.
"""
self._filelist = filelist
def find_file_by_name(self, filename):
"""Find a backed up file given its filename.
Return tuple (STAT, CONTREF, SIGREF, DELTAREF), where the
references may be None, or None instead of the entire tuple
if no file with the given name could be found.
"""
if self._filelist:
fc = self._filelist.find(filename)
if fc != None:
stat = fc.first_by_kind(obnam.cmp.STAT)
cont = fc.first_string_by_kind(obnam.cmp.CONTREF)
sig = fc.first_string_by_kind(obnam.cmp.SIGREF)
d = fc.first_string_by_kind(obnam.cmp.DELTAREF)
return obnam.cmp.parse_stat_component(stat), cont, sig, d
return None
def compute_signature(self, filename):
"""Compute rsync signature for a filename.
Return the identifier. Put the signature object in the queue to
be uploaded.
"""
sigdata = obnam.rsync.compute_signature(self._context, filename)
id = obnam.obj.object_id_new()
sig = obnam.obj.SignatureObject(id=id, sigdata=sigdata)
self.get_store().queue_object(sig)
return sig
def add_to_filegroup(self, fg, filename):
"""Add a file to a filegroup."""
self._context.progress.update_current_action(filename)
st = os.stat(filename)
if stat.S_ISREG(st.st_mode):
contref = obnam.io.create_file_contents_object(self._context,
filename)
sig = self.compute_signature(filename)
sigref = sig.get_id()
deltaref = None
else:
contref = None
sigref = None
deltaref = None
fg.add_file(os.path.basename(filename), st, contref, sigref, deltaref)
def make_filegroups(self, filenames):
"""Make list of new FILEGROUP objects.
Return list of object identifiers to the FILEGROUP objects.
"""
list = []
for filename in filenames:
if (not list or
len(list[-1].get_files()) >= MAX_PER_FILEGROUP):
id = obnam.obj.object_id_new()
list.append(obnam.obj.FileGroupObject(id=id))
self.add_to_filegroup(list[-1], filename)
self.get_store().queue_objects(list)
return list
def _make_absolute(self, basename, relatives):
return [os.path.join(basename, name) for name in relatives]
def backup_one_dir(self, dirname, subdirs, filenames):
"""Back up non-recursively one directory.
Return obnam.obj.DirObject that refers to the directory.
subdirs is the list of subdirectories (as DirObject) for this
directory.
"""
filenames = self._make_absolute(dirname, filenames)
filegroups = self.make_filegroups(filenames)
filegrouprefs = [fg.get_id() for fg in filegroups]
dirrefs = [subdir.get_id() for subdir in subdirs]
dir = obnam.obj.DirObject(id=obnam.obj.object_id_new(),
name=os.path.basename(dirname),
stat=os.stat(dirname),
dirrefs=dirrefs,
filegrouprefs=filegrouprefs)
self.get_store().queue_object(dir)
return dir
def backup_one_root(self, root):
"""Backup one root for the next generation."""
logging.debug("Backing up root %s" % root)
resolved = obnam.io.resolve(self._context, root)
logging.debug("Root resolves to %s" % resolved)
if not os.path.isdir(resolved):
raise obnam.ObnamException("Not a directory: %s" % root)
# FIXME: This needs to be able to handle non-directories, too!
subdirs_for_dir = {}
root_object = None
for tuple in obnam.walk.depth_first(resolved, prune=self.prune):
dirname, dirnames, filenames = tuple
logging.debug("Walked to directory %s" % dirname)
logging.debug(" with dirnames: %s" % dirnames)
logging.debug(" and filenames: %s" % filenames)
subdirs = subdirs_for_dir.get(dirname, [])
dir = self.backup_one_dir(dirname, subdirs, filenames)
if obnam.io.unsolve(self._context, dirname) != root:
parent = os.path.dirname(dirname)
if parent not in subdirs_for_dir:
subdirs_for_dir[parent] = []
subdirs_for_dir[parent].append(dir)
else:
root_object = dir
if dirname in subdirs_for_dir:
del subdirs_for_dir[dirname]
return root_object
def backup(self, roots):
"""Backup all the roots."""
start = int(time.time())
root_objs = []
for root in roots:
root_objs.append(self.backup_one_root(root))
end = int(time.time())
dirrefs = [o.get_id() for o in root_objs]
gen = obnam.obj.GenerationObject(id=obnam.obj.object_id_new(),
dirrefs=dirrefs, start=start,
end=end)
self.get_store().queue_object(gen)
return gen
|
