diff options
author | Lars Wirzenius <liw@liw.fi> | 2014-03-08 14:23:38 +0000 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2014-03-08 14:23:38 +0000 |
commit | 8f09b1807f879b3a8acbaf0f95aeb562d091fd48 (patch) | |
tree | 12849ad45cfb9a123f98b3c8f62e749da6a610f0 | |
parent | b146da5d96d863434aa2fdf21d5521c8690489b6 (diff) | |
parent | 6c999e971b58a3e69be926151e7224f6cdd3d89b (diff) | |
download | obnam-8f09b1807f879b3a8acbaf0f95aeb562d091fd48.tar.gz |
Merge branch 'liw/nemo-inis-bug-refactor'
Reported-by: Nemo Inis <nemoinis@hotmail.com>
-rw-r--r-- | NEWS | 5 | ||||
-rw-r--r-- | obnamlib/fmt_6/repo_fmt_6.py | 7 | ||||
-rw-r--r-- | obnamlib/plugins/backup_plugin.py | 7 | ||||
-rw-r--r-- | obnamlib/plugins/dump_repo_plugin.py | 119 | ||||
-rw-r--r-- | obnamlib/plugins/restore_plugin.py | 7 | ||||
-rw-r--r-- | without-tests | 1 | ||||
-rw-r--r-- | yarns/0050-multiple-clients.yarn | 16 |
7 files changed, 155 insertions, 7 deletions
@@ -62,6 +62,11 @@ Bug fixes: checkpoint was finished, instead of saying the name of the file. This is now fixed. +* Obnam now doesn't remove chunks that are shared between clients. + Previously, this would sometimes happen, because only the first + client would correctly record itself as using a chunk. Now all + clients do that. + Internal changes: * The `obnamlib.Error` exception class has been replaced by the diff --git a/obnamlib/fmt_6/repo_fmt_6.py b/obnamlib/fmt_6/repo_fmt_6.py index 90e04057..39050aa6 100644 --- a/obnamlib/fmt_6/repo_fmt_6.py +++ b/obnamlib/fmt_6/repo_fmt_6.py @@ -818,7 +818,12 @@ class RepositoryFormat6(obnamlib.RepositoryInterface): except obnamlib.RepositoryChunkDoesNotExist: return False actual_checksum = self._checksum(content) - expected_checksum = self._chunklist.get_checksum(chunk_id) + try: + expected_checksum = self._chunklist.get_checksum(chunk_id) + except KeyError: # pragma: no cover + # Chunk is not in the checksum tree, so we cannot valide + # its checksum. We'll just assume it's OK. + return True return actual_checksum == expected_checksum # Individual files in a generation. diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py index 75bd8d8d..c9900bee 100644 --- a/obnamlib/plugins/backup_plugin.py +++ b/obnamlib/plugins/backup_plugin.py @@ -890,6 +890,7 @@ class BackupPlugin(obnamlib.ObnamPlugin): for chunkid in find(): data2 = get(chunkid) if data == data2: + share(chunkid) return chunkid else: chunkid = put() @@ -898,11 +899,11 @@ class BackupPlugin(obnamlib.ObnamPlugin): elif mode == 'fatalist': existing = find() if existing: - return existing[0] + chunkid = existing[0] else: chunkid = put() - share(chunkid) - return chunkid + share(chunkid) + return chunkid else: if not hasattr(self, 'bad_deduplicate_reported'): logging.error('unknown --deduplicate setting value') diff --git a/obnamlib/plugins/dump_repo_plugin.py b/obnamlib/plugins/dump_repo_plugin.py new file mode 100644 index 00000000..7379ba39 --- /dev/null +++ b/obnamlib/plugins/dump_repo_plugin.py @@ -0,0 +1,119 @@ +# Copyright (C) 2014 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import yaml + +import obnamlib + + +class DumpRepositoryPlugin(obnamlib.ObnamPlugin): + + def enable(self): + self.app.add_subcommand( + 'dump-repo', self.cmd_dump_repo, arg_synopsis='') + + self.app.settings.boolean( + ['dump-repo-file-metadata'], + 'dump metadata about files?') + + def cmd_dump_repo(self, args): + repo = self.app.get_repository_object() + yaml.safe_dump_all( + self.dump_repository(repo), + stream=self.app.output, + default_flow_style=False) + repo.close() + + def dump_repository(self, repo): + yield self.dump_client_list(repo) + yield self.dump_chunk_ids(repo) + for client_name in repo.get_client_names(): + for x in self.dump_client(repo, client_name): + yield x + + def dump_client_list(self, repo): + return { + 'client-list': list(repo.get_client_names()), + } + + def dump_chunk_ids(self, repo): + return { + 'chunk-ids': list(repo.get_chunk_ids()), + } + + def dump_client(self, repo, client_name): + yield { + 'client-name': client_name, + 'encryption-key': repo.get_client_encryption_key_id(client_name), + 'client-keys': self.dump_client_keys(repo, client_name), + } + + for gen_id in repo.get_client_generation_ids(client_name): + yield self.dump_generation(repo, client_name, gen_id) + + def dump_client_keys(self, repo, client_name): + return dict( + (obnamlib.repo_key_name(key), + repo.get_client_key(client_name, key)) + for key in repo.get_allowed_client_keys() + if key != obnamlib.REPO_CLIENT_TEST_KEY + ) + + def dump_client_generation_ids(self, repo, client_name): + return [ + repo.make_generation_spec(gen_id) + for gen_id in repo.get_client_generation_ids(client_name) + ] + + def dump_generation(self, repo, client_name, gen_id): + obj = { + 'client-name': client_name, + 'generation-id': repo.make_generation_spec(gen_id), + 'generation-keys': self.dump_generation_keys(repo, gen_id), + 'generation-chunk-ids': + self.dump_generation_chunk_ids(repo, gen_id), + } + if self.app.settings['dump-repo-file-metadata']: + obj['files'] = self.dump_generation_files(repo, gen_id) + return obj + + def dump_generation_keys(self, repo, gen_id): + return dict( + (obnamlib.repo_key_name(key), + repo.get_generation_key(gen_id, key)) + for key in repo.get_allowed_generation_keys() + if key != obnamlib.REPO_GENERATION_TEST_KEY + ) + + def dump_generation_chunk_ids(self, repo, gen_id): + return repo.get_generation_chunk_ids(gen_id) + + def dump_generation_files(self, repo, gen_id): + result = {} + for filename in repo.walk_generation(gen_id, '/'): + result[filename] = { + 'file-chunk-ids': repo.get_file_chunk_ids(gen_id, filename), + 'file-keys': self.dump_file_keys(repo, gen_id, filename), + } + return result + + def dump_file_keys(self, repo, gen_id, filename): + return dict( + (obnamlib.repo_key_name(key), + repo.get_file_key(gen_id, filename, key)) + for key in repo.get_allowed_file_keys() + if key != obnamlib.REPO_FILE_TEST_KEY + ) diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py index ad9ba6f7..2456930c 100644 --- a/obnamlib/plugins/restore_plugin.py +++ b/obnamlib/plugins/restore_plugin.py @@ -237,13 +237,14 @@ class RestorePlugin(obnamlib.ObnamPlugin): msg = ('Could not set metadata: %s: %d: %s' % (pathname, e.errno, e.strerror)) logging.error(msg) - self.app.ts.notify(msg) + self.app.ts.error(msg) self.errors = True except Exception, e: - # Reaching this code path means we've hit a bug, so we log a full traceback. + # Reaching this code path means we've hit a bug, so we log + # a full traceback. msg = "Failed to restore %s:" % (pathname,) logging.exception(msg) - self.app.ts.notify(msg + " " + str(e)) + self.app.ts.error(msg + " " + str(e)) self.errors = True def restore_dir(self, gen, root, metadata): diff --git a/without-tests b/without-tests index b198157b..0cd579f5 100644 --- a/without-tests +++ b/without-tests @@ -24,3 +24,4 @@ test-plugins/hello_plugin.py test-plugins/oldhello_plugin.py test-plugins/aaa_hello_plugin.py test-plugins/wrongversion_plugin.py +obnamlib/plugins/dump_repo_plugin.py diff --git a/yarns/0050-multiple-clients.yarn b/yarns/0050-multiple-clients.yarn index 0cacd464..9119f034 100644 --- a/yarns/0050-multiple-clients.yarn +++ b/yarns/0050-multiple-clients.yarn @@ -53,3 +53,19 @@ existing client's name. GIVEN directory L with interesting filesystem objects WHEN user U1 backs up directory L to repository R THEN user U2 can see user U1 in repository R + +Two clients sharing chunks, one forgets its generations +------------------------------------------------------- + +What happens when two clients share chunks and one of them forgets its +generations? A problem was found in this scenario by Nemo Inis in +2014. + + SCENARIO two clients share chunks and one forgets is generations + GIVEN 1k of new data in directory L + AND a manifest of L in M + WHEN user U1 backs up directory L to repository R + AND user U2 backs up directory L to repository R + AND user U1 forgets the oldest generation in repository R + AND user U2 restores their latest generation in repository R into X + THEN L, restored to X, matches manifest M |