Fix repository corruption problems due to crashing

5 files changed, 109 insertions, 1 deletions

diff --git a/crash-test b/crash-test
new file mode 100755
index 00000000..812f3d14
--- /dev/null
+++ b/crash-test
@@ -0,0 +1,73 @@
+#!/bin/sh
+# Copyright 2012  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+set -eu
+
+if [ "$#" != 1 ]
+then
+    echo "usage: see source" 1>&2
+    exit 1
+fi
+
+N="$1"
+
+#tempdir="$(mktemp -d)"
+tempdir=t.tmp
+echo "Temporary directory: $tempdir"
+
+cat <<EOF > "$tempdir/conf"
+[config]
+repository = $tempdir/repo
+root = $tempdir/data
+log = $tempdir/obnam.log
+trace = larch
+crash-limit = $N
+checkpoint = 1m
+lock-timeout = 1
+EOF
+
+# Do a minimal backup to make sure the repository works at least once.
+mkdir "$tempdir/data"
+./obnam backup --no-default-config --config "$tempdir/conf"
+
+genbackupdata --create=100m "$tempdir/data"
+
+while true
+do
+    rm -f "$tempdir/obnam.log"
+
+    echo "Trying backup with at most $N writes to repository"
+    find "$tempdir/repo" -name lock -delete
+    if ./obnam backup --no-default-config --config "$tempdir/conf" 2>/dev/null
+    then
+        echo "Backup finished ok, done"
+        break
+    fi
+
+    if ! grep -q '^Exception: Crashing as requested' "$tempdir/obnam.log"
+    then
+        echo "Backup terminated because of unrequested crash" 1>&2
+        exit 1
+    fi
+
+#    ./obnam fsck --no-default-config --config "$tempdir/conf" || true
+done
+
+rm -rf "$tempdir"
+
+echo "OK"
+
diff --git a/obnamlib/app.py b/obnamlib/app.py
index 4bfded83..f9d2ba7a 100644
--- a/obnamlib/app.py
+++ b/obnamlib/app.py
@@ -91,6 +91,14 @@ class App(cliapp.Application):
                               metavar='TIMEOUT',
                               default=60)
 
+        self.settings.integer(['crash-limit'],
+                              'artificially crash the program after COUNTER '
+                                'files written to the repository; this is '
+                                'useful for crash testing the application, '
+                                'and should not be enabled for real use; '
+                                'set to 0 to disable (disabled by default)',
+                              metavar='COUNTER')
+
         # The following needs to be done here, because it needs
         # to be done before option processing. This is a bit ugly,
         # but the best we can do with the current cliapp structure.
@@ -165,6 +173,8 @@ class App(cliapp.Application):
         logging.debug('opening repository (create=%s)' % create)
         repopath = self.settings['repository']
         repofs = self.fsf.new(repopath, create=create)
+        if self.settings['crash-limit'] > 0:
+            repofs.crash_limit = self.settings['crash-limit']
         repofs.connect()
         return obnamlib.Repository(repofs, 
                                     self.settings['node-size'],
diff --git a/obnamlib/lockmgr.py b/obnamlib/lockmgr.py
index d5fccba8..408b2b01 100644
--- a/obnamlib/lockmgr.py
+++ b/obnamlib/lockmgr.py
@@ -50,7 +50,7 @@ class LockManager(object):
                 self._fs.lock(self._lockname(dirname))
             except obnamlib.LockFail:
                 if self._time() - now >= self.timeout:
-                    raise obnamlib.LockFail()
+                    raise obnamlib.LockFail('Lock timeout')
             else:
                 return
             self._sleep()
diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py
index 00ca1f86..7b191174 100644
--- a/obnamlib/vfs_local.py
+++ b/obnamlib/vfs_local.py
@@ -57,6 +57,18 @@ class LocalFS(obnamlib.VirtualFileSystem):
         obnamlib.VirtualFileSystem.__init__(self, baseurl)
         self.reinit(baseurl, create=create)
 
+        # For testing purposes, allow setting a limit on write operations
+        # after which an exception gets raised. If set to None, no crash.
+        self.crash_limit = None
+        self.crash_counter = 0
+
+    def maybe_crash(self): # pragma: no cover
+        if self.crash_limit is not None:
+            self.crash_counter += 1
+            if self.crash_counter >= self.crash_limit:
+                raise Exception('Crashing as requested after %d writes' %
+                                    self.crash_counter)
+
     def reinit(self, baseurl, create=False):
         # We fake chdir so that it doesn't mess with the caller's 
         # perception of current working directory. This also benefits
@@ -108,10 +120,12 @@ class LocalFS(obnamlib.VirtualFileSystem):
     def remove(self, pathname):
         tracing.trace('remove %s', pathname)
         os.remove(self.join(pathname))
+        self.maybe_crash()
 
     def rename(self, old, new):
         tracing.trace('rename %s %s', old, new)
         os.rename(self.join(old), self.join(new))
+        self.maybe_crash()
 
     def lstat(self, pathname):
         (ret, dev, ino, mode, nlink, uid, gid, rdev, size, blksize, blocks,
@@ -185,6 +199,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
         tracing.trace('existing=%s', existing)
         tracing.trace('new=%s', new)
         os.link(self.join(existing), self.join(new))
+        self.maybe_crash()
 
     def readlink(self, pathname):
         return os.readlink(self.join(pathname))
@@ -193,6 +208,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
         tracing.trace('existing=%s', existing)
         tracing.trace('new=%s', new)
         os.symlink(existing, self.join(new))
+        self.maybe_crash()
 
     def open(self, pathname, mode):
         tracing.trace('pathname=%s', pathname)
@@ -223,14 +239,17 @@ class LocalFS(obnamlib.VirtualFileSystem):
     def mkdir(self, pathname):
         tracing.trace('mkdir %s', pathname)
         os.mkdir(self.join(pathname))
+        self.maybe_crash()
 
     def makedirs(self, pathname):
         tracing.trace('makedirs %s', pathname)
         os.makedirs(self.join(pathname))
+        self.maybe_crash()
 
     def rmdir(self, pathname):
         tracing.trace('rmdir %s', pathname)
         os.rmdir(self.join(pathname))
+        self.maybe_crash()
 
     def cat(self, pathname):
         pathname = self.join(pathname)
@@ -256,6 +275,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
             os.remove(tempname)
             raise
         os.remove(tempname)
+        self.maybe_crash()
 
     def overwrite_file(self, pathname, contents, make_backup=True):
         tracing.trace('overwrite_file %s', pathname)
@@ -279,6 +299,8 @@ class LocalFS(obnamlib.VirtualFileSystem):
                 os.remove(bak)
             except OSError:
                 pass
+
+        self.maybe_crash()
                 
     def _write_to_tempfile(self, pathname, contents):
         path = self.join(pathname)
diff --git a/setup.py b/setup.py
index 26dd2e05..b69164d5 100644
--- a/setup.py
+++ b/setup.py
@@ -90,6 +90,9 @@ class Check(Command):
         subprocess.check_call(['./test-locking', num_clients, num_generations,
                                test_repo, test_repo])
 
+        print "run crash test"
+        subprocess.check_call(['./crash-test', '100'])
+
         if self.network:
             print "run sftp tests"
             subprocess.check_call(['./test-sftpfs'])