Merge branch 'liw/nemo-inis-bug-refactor'

Reported-by: Nemo Inis <nemoinis@hotmail.com>

7 files changed, 155 insertions, 7 deletions

diff --git a/NEWS b/NEWS
index 60bf34f9..27cd8ec3 100644
--- a/NEWS
+++ b/NEWS
@@ -62,6 +62,11 @@ Bug fixes:
   checkpoint was finished, instead of saying the name of the file.
   This is now fixed.
 
+* Obnam now doesn't remove chunks that are shared between clients.
+  Previously, this would sometimes happen, because only the first
+  client would correctly record itself as using a chunk. Now all
+  clients do that.
+
 Internal changes:
 
 * The `obnamlib.Error` exception class has been replaced by the
diff --git a/obnamlib/fmt_6/repo_fmt_6.py b/obnamlib/fmt_6/repo_fmt_6.py
index 90e04057..39050aa6 100644
--- a/obnamlib/fmt_6/repo_fmt_6.py
+++ b/obnamlib/fmt_6/repo_fmt_6.py
@@ -818,7 +818,12 @@ class RepositoryFormat6(obnamlib.RepositoryInterface):
         except obnamlib.RepositoryChunkDoesNotExist:
             return False
         actual_checksum = self._checksum(content)
-        expected_checksum = self._chunklist.get_checksum(chunk_id)
+        try:
+            expected_checksum = self._chunklist.get_checksum(chunk_id)
+        except KeyError: # pragma: no cover
+            # Chunk is not in the checksum tree, so we cannot valide
+            # its checksum. We'll just assume it's OK.
+            return True
         return actual_checksum == expected_checksum
 
     # Individual files in a generation.
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index 75bd8d8d..c9900bee 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -890,6 +890,7 @@ class BackupPlugin(obnamlib.ObnamPlugin):
             for chunkid in find():
                 data2 = get(chunkid)
                 if data == data2:
+                    share(chunkid)
                     return chunkid
             else:
                 chunkid = put()
@@ -898,11 +899,11 @@ class BackupPlugin(obnamlib.ObnamPlugin):
         elif mode == 'fatalist':
             existing = find()
             if existing:
-                return existing[0]
+                chunkid = existing[0]
             else:
                 chunkid = put()
-                share(chunkid)
-                return chunkid
+            share(chunkid)
+            return chunkid
         else:
             if not hasattr(self, 'bad_deduplicate_reported'):
                 logging.error('unknown --deduplicate setting value')
diff --git a/obnamlib/plugins/dump_repo_plugin.py b/obnamlib/plugins/dump_repo_plugin.py
new file mode 100644
index 00000000..7379ba39
--- /dev/null
+++ b/obnamlib/plugins/dump_repo_plugin.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2014  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import yaml
+
+import obnamlib
+
+
+class DumpRepositoryPlugin(obnamlib.ObnamPlugin):
+
+    def enable(self):
+        self.app.add_subcommand(
+            'dump-repo', self.cmd_dump_repo, arg_synopsis='')
+
+        self.app.settings.boolean(
+            ['dump-repo-file-metadata'],
+            'dump metadata about files?')
+
+    def cmd_dump_repo(self, args):
+        repo = self.app.get_repository_object()
+        yaml.safe_dump_all(
+            self.dump_repository(repo),
+            stream=self.app.output,
+            default_flow_style=False)
+        repo.close()
+
+    def dump_repository(self, repo):
+        yield self.dump_client_list(repo)
+        yield self.dump_chunk_ids(repo)
+        for client_name in repo.get_client_names():
+            for x in self.dump_client(repo, client_name):
+                yield x
+
+    def dump_client_list(self, repo):
+        return {
+            'client-list': list(repo.get_client_names()),
+            }
+
+    def dump_chunk_ids(self, repo):
+        return {
+            'chunk-ids': list(repo.get_chunk_ids()),
+            }
+
+    def dump_client(self, repo, client_name):
+        yield {
+            'client-name': client_name,
+            'encryption-key': repo.get_client_encryption_key_id(client_name),
+            'client-keys': self.dump_client_keys(repo, client_name),
+            }
+
+        for gen_id in repo.get_client_generation_ids(client_name):
+            yield self.dump_generation(repo, client_name, gen_id)
+
+    def dump_client_keys(self, repo, client_name):
+        return dict(
+            (obnamlib.repo_key_name(key),
+             repo.get_client_key(client_name, key))
+            for key in repo.get_allowed_client_keys()
+            if key != obnamlib.REPO_CLIENT_TEST_KEY
+            )
+
+    def dump_client_generation_ids(self, repo, client_name):
+        return [
+            repo.make_generation_spec(gen_id)
+            for gen_id in repo.get_client_generation_ids(client_name)
+            ]
+
+    def dump_generation(self, repo, client_name, gen_id):
+        obj = {
+            'client-name': client_name,
+            'generation-id': repo.make_generation_spec(gen_id),
+            'generation-keys': self.dump_generation_keys(repo, gen_id),
+            'generation-chunk-ids':
+                self.dump_generation_chunk_ids(repo, gen_id),
+            }
+        if self.app.settings['dump-repo-file-metadata']:
+            obj['files'] = self.dump_generation_files(repo, gen_id)
+        return obj
+
+    def dump_generation_keys(self, repo, gen_id):
+        return dict(
+            (obnamlib.repo_key_name(key),
+             repo.get_generation_key(gen_id, key))
+            for key in repo.get_allowed_generation_keys()
+            if key != obnamlib.REPO_GENERATION_TEST_KEY
+            )
+
+    def dump_generation_chunk_ids(self, repo, gen_id):
+        return repo.get_generation_chunk_ids(gen_id)
+
+    def dump_generation_files(self, repo, gen_id):
+        result = {}
+        for filename in repo.walk_generation(gen_id, '/'):
+            result[filename] = {
+                'file-chunk-ids': repo.get_file_chunk_ids(gen_id, filename),
+                'file-keys': self.dump_file_keys(repo, gen_id, filename),
+                }
+        return result
+
+    def dump_file_keys(self, repo, gen_id, filename):
+        return dict(
+            (obnamlib.repo_key_name(key),
+             repo.get_file_key(gen_id, filename, key))
+            for key in repo.get_allowed_file_keys()
+            if key != obnamlib.REPO_FILE_TEST_KEY
+            )
diff --git a/obnamlib/plugins/restore_plugin.py b/obnamlib/plugins/restore_plugin.py
index ad9ba6f7..2456930c 100644
--- a/obnamlib/plugins/restore_plugin.py
+++ b/obnamlib/plugins/restore_plugin.py
@@ -237,13 +237,14 @@ class RestorePlugin(obnamlib.ObnamPlugin):
                     msg = ('Could not set metadata: %s: %d: %s' %
                             (pathname, e.errno, e.strerror))
                     logging.error(msg)
-                    self.app.ts.notify(msg)
+                    self.app.ts.error(msg)
                     self.errors = True
         except Exception, e:
-            # Reaching this code path means we've hit a bug, so we log a full traceback.
+            # Reaching this code path means we've hit a bug, so we log
+            # a full traceback.
             msg = "Failed to restore %s:" % (pathname,)
             logging.exception(msg)
-            self.app.ts.notify(msg + " " + str(e))
+            self.app.ts.error(msg + " " + str(e))
             self.errors = True
 
     def restore_dir(self, gen, root, metadata):
diff --git a/without-tests b/without-tests
index b198157b..0cd579f5 100644
--- a/without-tests
+++ b/without-tests
@@ -24,3 +24,4 @@ test-plugins/hello_plugin.py
 test-plugins/oldhello_plugin.py
 test-plugins/aaa_hello_plugin.py
 test-plugins/wrongversion_plugin.py
+obnamlib/plugins/dump_repo_plugin.py
diff --git a/yarns/0050-multiple-clients.yarn b/yarns/0050-multiple-clients.yarn
index 0cacd464..9119f034 100644
--- a/yarns/0050-multiple-clients.yarn
+++ b/yarns/0050-multiple-clients.yarn
@@ -53,3 +53,19 @@ existing client's name.
     GIVEN directory L with interesting filesystem objects
     WHEN user U1 backs up directory L to repository R
     THEN user U2 can see user U1 in repository R
+
+Two clients sharing chunks, one forgets its generations
+-------------------------------------------------------
+
+What happens when two clients share chunks and one of them forgets its
+generations? A problem was found in this scenario by Nemo Inis in
+2014.
+
+    SCENARIO two clients share chunks and one forgets is generations
+    GIVEN 1k of new data in directory L
+    AND a manifest of L in M
+    WHEN user U1 backs up directory L to repository R
+    AND user U2 backs up directory L to repository R
+    AND user U1 forgets the oldest generation in repository R
+    AND user U2 restores their latest generation in repository R into X
+    THEN L, restored to X, matches manifest M