Add repository locking between clients

19 files changed, 570 insertions, 87 deletions

diff --git a/NEWS b/NEWS
index ad0f0be0..140a8b30 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,13 @@ Obnam NEWS
 
 This file summarizes changes between releases of Obnam.
 
+Version 0.26, released UNRELEASED; a BETA release
+-------------------------------------------------
+
+* Clients now lock the parts of the backup repository they're using,
+  while making any changes, so that multiple clients can work at the
+  same time without corrupting the repository.
+
 Version 0.25, released 2012-02-18; a BETA release
 -------------------------------------------------
 
diff --git a/_obnammodule.c b/_obnammodule.c
index 42d9847a..5e377e97 100644
--- a/_obnammodule.c
+++ b/_obnammodule.c
@@ -74,16 +74,22 @@ utimensat_wrapper(PyObject *self, PyObject *args)
 {
     int ret;
     const char *filename;
+    long atime_sec, atime_nsec;
+    long mtime_sec, mtime_nsec;
     struct timespec tv[2];
 
     if (!PyArg_ParseTuple(args, "sllll", 
                           &filename, 
-                          &tv[0].tv_sec,
-                          &tv[0].tv_nsec,
-                          &tv[1].tv_sec,
-                          &tv[1].tv_nsec))
+                          &atime_sec,
+                          &atime_nsec,
+                          &mtime_sec,
+                          &mtime_nsec))
         return NULL;
 
+    tv[0].tv_sec = atime_sec;
+    tv[0].tv_nsec = atime_nsec;
+    tv[1].tv_sec = mtime_sec;
+    tv[1].tv_nsec = mtime_nsec;
     ret = utimensat(AT_FDCWD, filename, tv, AT_SYMLINK_NOFOLLOW);
     if (ret == -1)
         ret = errno;
diff --git a/dumpobjs b/dumpobjs
index 04254a27..8469d86b 100755
--- a/dumpobjs
+++ b/dumpobjs
@@ -17,6 +17,7 @@
 
 import os
 import sys
+import time
 
 import obnamlib
 
@@ -31,7 +32,8 @@ repo = obnamlib.Repository(fs, obnamlib.DEFAULT_NODE_SIZE,
                        obnamlib.DEFAULT_UPLOAD_QUEUE_SIZE, None,
                        obnamlib.IDPATH_DEPTH,
                        obnamlib.IDPATH_BITS,
-                       obnamlib.IDPATH_SKIP)
+                       obnamlib.IDPATH_SKIP,
+                       time.time, 0)
 for objid in find_objids(fs):
     obj = repo.get_object(objid)
     print 'id %s (%s):' % (obj.id, obj.__class__.__name__)
diff --git a/lock-and-increment b/lock-and-increment
new file mode 100755
index 00000000..d7b945c1
--- /dev/null
+++ b/lock-and-increment
@@ -0,0 +1,41 @@
+#!/usr/bin/python
+# Copyright 2012  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import sys
+
+import obnamlib
+
+
+dirname = sys.argv[1]
+count_to = int(sys.argv[2])
+filename = os.path.join(dirname, 'counter')
+
+fs = obnamlib.LocalFS('/')
+lm = obnamlib.LockManager(fs, 60)
+
+for i in range(count_to):
+    lm.lock([dirname])
+    if fs.exists(filename):
+        data = fs.cat(filename)
+        counter = int(data)
+        counter += 1
+        fs.overwrite_file(filename, str(counter))
+    else:
+        fs.write_file(filename, str(1))
+    lm.unlock([dirname])
+
diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py
index 8615a938..07b68e54 100644
--- a/obnamlib/__init__.py
+++ b/obnamlib/__init__.py
@@ -83,6 +83,7 @@ from chunklist import ChunkList
 from clientlist import ClientList
 from checksumtree import ChecksumTree
 from clientmetadatatree import ClientMetadataTree
+from lockmgr import LockManager
 from repo import Repository, LockFail, BadFormat
 from forget_policy import ForgetPolicy
 from app import App
diff --git a/obnamlib/app.py b/obnamlib/app.py
index 4145a244..4bfded83 100644
--- a/obnamlib/app.py
+++ b/obnamlib/app.py
@@ -84,6 +84,13 @@ class App(cliapp.Application):
                                 'this is only useful for testing purposes',
                              metavar='TIMESTAMP')
 
+        self.settings.integer(['lock-timeout'],
+                              'when locking in the backup repository, '
+                                'wait TIMEOUT seconds for an existing lock '
+                                'to go away before giving up',
+                              metavar='TIMEOUT',
+                              default=60)
+
         # The following needs to be done here, because it needs
         # to be done before option processing. This is a bit ugly,
         # but the best we can do with the current cliapp structure.
@@ -119,7 +126,7 @@ class App(cliapp.Application):
         # we create one instance here, which will immediately be destroyed.
         # FIXME: This is fugly.
         obnamlib.Repository(None, 1000, 1000, 100, self.hooks, 10, 10, 10,
-                            self.time)
+                            self.time, 0)
 
     def plugins_dir(self):
         return os.path.join(os.path.dirname(obnamlib.__file__), 'plugins')
@@ -167,7 +174,8 @@ class App(cliapp.Application):
                                     self.settings['idpath-depth'],
                                     self.settings['idpath-bits'],
                                     self.settings['idpath-skip'],
-                                    self.time)
+                                    self.time,
+                                    self.settings['lock-timeout'])
 
     def time(self):
         '''Return current time in seconds since epoch.
diff --git a/obnamlib/lockmgr.py b/obnamlib/lockmgr.py
new file mode 100644
index 00000000..d5fccba8
--- /dev/null
+++ b/obnamlib/lockmgr.py
@@ -0,0 +1,77 @@
+# Copyright 2012  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import time
+
+import obnamlib
+
+
+class LockManager(object):
+
+    '''Lock and unlock sets of directories at once.'''
+    
+    def __init__(self, fs, timeout):
+        self._fs = fs
+        self.timeout = timeout
+        
+    def _time(self): # pragma: no cover
+        return time.time()
+        
+    def _sleep(self): # pragma: no cover
+        time.sleep(1)
+        
+    def sort(self, dirnames):
+        def bytelist(s):
+            return [ord(s) for s in str(s)]
+        return sorted(dirnames, key=bytelist)
+
+    def _lockname(self, dirname):
+        return os.path.join(dirname, 'lock')
+
+        
+    def _lock_one(self, dirname):
+        now = self._time()
+        while True:
+            try:
+                self._fs.lock(self._lockname(dirname))
+            except obnamlib.LockFail:
+                if self._time() - now >= self.timeout:
+                    raise obnamlib.LockFail()
+            else:
+                return
+            self._sleep()
+        
+    def _unlock_one(self, dirname):
+        self._fs.unlock(self._lockname(dirname))
+        
+    def lock(self, dirnames):
+        '''Lock ALL the directories.'''
+        we_locked = []
+        for dirname in self.sort(dirnames):
+            try:
+                self._lock_one(dirname)
+            except obnamlib.LockFail:
+                self.unlock(we_locked)
+                raise
+            else:
+                we_locked.append(dirname)
+
+    def unlock(self, dirnames):
+        '''Unlock ALL the directories.'''
+        for dirname in self.sort(dirnames):
+            self._unlock_one(dirname)
+
diff --git a/obnamlib/lockmgr_tests.py b/obnamlib/lockmgr_tests.py
new file mode 100644
index 00000000..83942d24
--- /dev/null
+++ b/obnamlib/lockmgr_tests.py
@@ -0,0 +1,93 @@
+# Copyright 2012  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import shutil
+import tempfile
+import unittest
+
+import obnamlib
+
+
+class LockManagerTests(unittest.TestCase):
+
+    def locked(self, dirname):
+        return os.path.exists(os.path.join(dirname, 'lock'))
+
+    def fake_time(self):
+        self.now += 1
+        return self.now
+
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.dirnames = []
+        for x in ['a', 'b', 'c']:
+            dirname = os.path.join(self.tempdir, x)
+            os.mkdir(dirname)
+            self.dirnames.append(dirname)
+        self.fs = obnamlib.LocalFS(self.tempdir)
+        self.timeout = 10
+        self.now = 0
+        self.lm = obnamlib.LockManager(self.fs, self.timeout)
+        self.lm._time = self.fake_time
+        self.lm._sleep = lambda: None
+        
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+        
+    def test_has_nothing_locked_initially(self):
+        for dirname in self.dirnames:
+            self.assertFalse(self.locked(dirname))
+
+    def test_locks_single_directory(self):
+        self.lm.lock([self.dirnames[0]])
+        self.assertTrue(self.locked(self.dirnames[0]))
+
+    def test_unlocks_single_directory(self):
+        self.lm.lock([self.dirnames[0]])
+        self.lm.unlock([self.dirnames[0]])
+        self.assertFalse(self.locked(self.dirnames[0]))
+
+    def test_waits_until_timeout_for_locked_directory(self):
+        self.lm.lock([self.dirnames[0]])
+        self.assertRaises(obnamlib.LockFail,
+                          self.lm.lock, [self.dirnames[0]])
+
+    def test_notices_when_preexisting_lock_goes_away(self):
+        self.lm.lock([self.dirnames[0]])
+        self.lm._sleep = lambda: os.remove(self.lm._lockname(self.dirnames[0]))
+        self.lm.lock([self.dirnames[0]])
+        self.assertTrue(True)
+
+    def test_locks_all_directories(self):
+        self.lm.lock(self.dirnames)
+        for dirname in self.dirnames:
+            self.assertTrue(self.locked(dirname))
+
+    def test_unlocks_all_directories(self):
+        self.lm.lock(self.dirnames)
+        self.lm.unlock(self.dirnames)
+        for dirname in self.dirnames:
+            self.assertFalse(self.locked(dirname))
+
+    def test_does_not_lock_anything_if_one_lock_fails(self):
+        self.lm.lock([self.dirnames[-1]])
+        self.assertRaises(obnamlib.LockFail, self.lm.lock, self.dirnames)
+        for dirname in self.dirnames[:-1]:
+            self.assertFalse(self.locked(dirname))
+        self.assertTrue(self.locked(self.dirnames[-1]))
+
+
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index 50e279c1..f253af9c 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -172,9 +172,10 @@ class BackupPlugin(obnamlib.ObnamPlugin):
             self.repo = self.app.open_repository()
             self.repo.open_client(client_name)
         else:
-            self.repo = self.app.open_repository(create=True)
             self.add_client(client_name)
+            self.repo = self.app.open_repository()
             self.repo.lock_client(client_name)
+            self.repo.lock_shared()
 
         self.started = time.time()
         self.errors = False
@@ -188,6 +189,7 @@ class BackupPlugin(obnamlib.ObnamPlugin):
             self.app.ts['what'] = 'committing changes;'
             if not self.pretend:
                 self.repo.commit_client()
+                self.repo.commit_shared()
             self.repo.fs.close()
             self.app.ts.clear()
             self.report_stats()
@@ -198,14 +200,20 @@ class BackupPlugin(obnamlib.ObnamPlugin):
             if self.repo.got_client_lock:
                 logging.info('Unlocking client because of error')
                 self.repo.unlock_client()
+                self.repo.unlock_shared()
             raise
 
     def add_client(self, client_name):
-        if client_name not in self.repo.list_clients():
+        repo = self.app.open_repository(create=True)
+        repo.lock_root()
+        if client_name not in repo.list_clients():
             tracing.trace('adding new client %s' % client_name)
-            self.repo.lock_root()
-            self.repo.add_client(client_name)
-            self.repo.commit_root()
+            tracing.trace('client list before adding: %s' % 
+                            repo.list_clients())
+            repo.add_client(client_name)
+            tracing.trace('client list after adding: %s' % 
+                            repo.list_clients())
+        repo.commit_root()
 
     def compile_exclusion_patterns(self):
         log = self.app.settings['log']
@@ -289,7 +297,10 @@ class BackupPlugin(obnamlib.ObnamPlugin):
             self.checkpoints.append(self.repo.new_generation)
             self.backup_parents('.')
             self.repo.commit_client(checkpoint=True)
+            self.repo.commit_shared()
+            self.repo = self.app.open_repository()
             self.repo.lock_client(self.app.settings['client-name'])
+            self.repo.lock_shared()
             self.repo.start_generation()
             self.last_checkpoint = self.repo.fs.bytes_written
             self.app.dump_memory_profile('at end of checkpoint')
diff --git a/obnamlib/plugins/forget_plugin.py b/obnamlib/plugins/forget_plugin.py
index a7d7ed85..8102635e 100644
--- a/obnamlib/plugins/forget_plugin.py
+++ b/obnamlib/plugins/forget_plugin.py
@@ -41,6 +41,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin):
 
         self.repo = self.app.open_repository()
         self.repo.lock_client(self.app.settings['client-name'])
+        self.repo.lock_shared()
 
         self.app.dump_memory_profile('at beginning')
         if args:
@@ -72,6 +73,7 @@ class ForgetPlugin(obnamlib.ObnamPlugin):
                 self.app.dump_memory_profile('after removing %s' % genid)
 
         self.repo.commit_client()
+        self.repo.commit_shared()
         self.app.dump_memory_profile('after committing')
         self.repo.fs.close()
         self.app.ts.finish()
diff --git a/obnamlib/plugins/sftp_plugin.py b/obnamlib/plugins/sftp_plugin.py
index 00342f53..082e74df 100644
--- a/obnamlib/plugins/sftp_plugin.py
+++ b/obnamlib/plugins/sftp_plugin.py
@@ -360,7 +360,7 @@ class SftpFS(obnamlib.VirtualFileSystem):
             self.write_file(lockname, '')
         except IOError, e:
             if e.errno == errno.EEXIST:
-                raise obnamlib.Error('Lock %s already exists' % lockname)
+                raise obnamlib.LockFail('Lock %s already exists' % lockname)
             else:
                 raise
 
diff --git a/obnamlib/repo.py b/obnamlib/repo.py
index a25a849f..ee40090b 100644
--- a/obnamlib/repo.py
+++ b/obnamlib/repo.py
@@ -64,7 +64,10 @@ class HookedFS(object):
         toplevel = self._get_toplevel(filename)
         return self.hooks.call('repository-read-data', data,
                                 repo=self.repo, toplevel=toplevel)
-        
+
+    def lock(self, filename):
+        self.fs.lock(filename)
+
     def write_file(self, filename, data):
         tracing.trace('writing hooked %s' % filename)
         toplevel = self._get_toplevel(filename)
@@ -129,7 +132,8 @@ class Repository(object):
     format_version = 5
 
     def __init__(self, fs, node_size, upload_queue_size, lru_size, hooks,
-                 idpath_depth, idpath_bits, idpath_skip, current_time):
+                 idpath_depth, idpath_bits, idpath_skip, current_time,
+                 lock_timeout):
 
         self.current_time = current_time
         self.setup_hooks(hooks or obnamlib.HookManager())
@@ -137,12 +141,13 @@ class Repository(object):
         self.node_size = node_size
         self.upload_queue_size = upload_queue_size
         self.lru_size = lru_size
+        self.lockmgr = obnamlib.LockManager(self.fs, lock_timeout)
         self.got_root_lock = False
         self.clientlist = obnamlib.ClientList(self.fs, node_size, 
                                               upload_queue_size, 
                                               lru_size, self)
+        self.got_shared_lock = False
         self.got_client_lock = False
-        self.client_lockfile = None
         self.current_client = None
         self.current_client_id = None
         self.new_generation = None
@@ -211,10 +216,15 @@ class Repository(object):
         if not self.got_root_lock:
             raise LockFail('have not got lock on root node')
 
+    def require_shared_lock(self):
+        '''Ensure we have the lock on the shared B-trees except clientlist.'''
+        if not self.got_shared_lock:
+            raise LockFail('have not got lock on shared B-trees')
+
     def require_client_lock(self):
         '''Ensure we have the lock on the currently open client.'''
         if not self.got_client_lock:
-                raise LockFail('have not got lock on client')
+            raise LockFail('have not got lock on client')
 
     def require_open_client(self):
         '''Ensure we have opened the client (r/w or r/o).'''
@@ -226,6 +236,21 @@ class Repository(object):
         if self.new_generation is None:
             raise obnamlib.Error('new generation has not started')
 
+    def require_no_root_lock(self):
+        '''Ensure we haven't locked root yet.'''
+        if self.got_root_lock:
+            raise obnamlib.Error('We have already locked root, oops')
+
+    def require_no_shared_lock(self):
+        '''Ensure we haven't locked shared B-trees yet.'''
+        if self.got_shared_lock:
+            raise obnamlib.Error('We have already locked shared B-trees, oops')
+
+    def require_no_client_lock(self):
+        '''Ensure we haven't locked the per-client B-tree yet.'''
+        if self.got_client_lock:
+            raise obnamlib.Error('We have already locked the client, oops')
+
     def lock_root(self):
         '''Lock root node.
         
@@ -234,13 +259,13 @@ class Repository(object):
         
         '''
 
-        tracing.trace('locking root')        
+        tracing.trace('locking root')
+        self.require_no_root_lock()
+        self.require_no_client_lock()
+        self.require_no_shared_lock()
+
+        self.lockmgr.lock(['.'])
         self.check_format_version()
-        try:
-            self.fs.fs.write_file('root.lock', '')
-        except OSError, e:
-            if e.errno == errno.EEXIST:
-                raise LockFail('Lock file root.lock already exists')
         self.got_root_lock = True
         self.added_clients = []
         self.removed_clients = []
@@ -252,7 +277,7 @@ class Repository(object):
         self.require_root_lock()
         self.added_clients = []
         self.removed_clients = []
-        self.fs.remove('root.lock')
+        self.lockmgr.unlock(['.'])
         self.got_root_lock = False
         
     def commit_root(self):
@@ -338,6 +363,43 @@ class Repository(object):
         if client_name not in self.list_clients():
             raise obnamlib.Error('client %s does not exist' % client_name)
         self.removed_clients.append(client_name)
+
+    @property
+    def shared_dirs(self):
+        return [self.chunklist.dirname, self.chunksums.dirname]
+        
+    def lock_shared(self):
+        '''Lock a client for exclusive write access.
+        
+        Raise obnamlib.LockFail if locking fails. Lock will be released
+        by commit_client() or unlock_client().
+
+        '''
+
+        tracing.trace('locking shared')
+        self.require_no_shared_lock()
+        self.check_format_version()
+        self.lockmgr.lock(self.shared_dirs)
+        self.got_shared_lock = True
+        tracing.trace('starting changes in chunksums and chunklist')
+        self.chunksums.start_changes()
+        self.chunklist.start_changes()
+
+    def commit_shared(self):
+        '''Commit changes to shared B-trees.'''
+        
+        tracing.trace('committing shared')
+        self.require_shared_lock()
+        self.chunklist.commit()
+        self.chunksums.commit()
+        self.unlock_shared()
+
+    def unlock_shared(self):
+        '''Unlock currently locked shared B-trees.'''
+        tracing.trace('unlocking shared')
+        self.require_shared_lock()
+        self.lockmgr.unlock(self.shared_dirs)
+        self.got_shared_lock = False
         
     def lock_client(self, client_name):
         '''Lock a client for exclusive write access.
@@ -348,6 +410,9 @@ class Repository(object):
         '''
 
         tracing.trace('client_name=%s', client_name)
+        self.require_no_client_lock()
+        self.require_no_shared_lock()
+        
         self.check_format_version()
         client_id = self.clientlist.get_client_id(client_name)
         if client_id is None:
@@ -357,14 +422,9 @@ class Repository(object):
         if not self.fs.exists(client_dir):
             self.fs.mkdir(client_dir)
             self.hooks.call('repository-toplevel-init', self, client_dir)
-        lockname = os.path.join(client_dir, 'lock')
-        try:
-            self.fs.write_file(lockname, '')
-        except OSError, e:
-            if e.errno == errno.EEXIST:
-                raise LockFail('client %s is already locked' % client_name)
+
+        self.lockmgr.lock([client_dir])
         self.got_client_lock = True
-        self.client_lockfile = lockname
         self.current_client = client_name
         self.current_client_id = client_id
         self.added_generations = []
@@ -382,11 +442,10 @@ class Repository(object):
         self.new_generation = None
         for genid in self.added_generations:
             self._really_remove_generation(genid)
+        self.lockmgr.unlock([self.client.dirname])
         self.client = None # FIXME: This should remove uncommitted data.
         self.added_generations = []
         self.removed_generations = []
-        self.fs.remove(self.client_lockfile)
-        self.client_lockfile = None
         self.got_client_lock = False
         self.current_client = None
         self.current_client_id = None
@@ -395,14 +454,13 @@ class Repository(object):
         '''Commit changes to and unlock currently locked client.'''
         tracing.trace('committing client (checkpoint=%s)', checkpoint)
         self.require_client_lock()
+        self.require_shared_lock()
         if self.new_generation:
             self.client.set_current_generation_is_checkpoint(checkpoint)
         self.added_generations = []
         for genid in self.removed_generations:
             self._really_remove_generation(genid)
         self.client.commit()
-        self.chunklist.commit()
-        self.chunksums.commit()
         self.unlock_client()
         
     def open_client(self, client_name):
@@ -477,6 +535,7 @@ class Repository(object):
                     self.remove_chunk(chunk_id)
 
         self.require_client_lock()
+        self.require_shared_lock()
         logging.debug('_really_remove_generation: %d' % gen_id)
         chunk_ids = set(self.client.list_chunks_in_generation(gen_id))
         chunk_ids = filter_away_chunks_used_by_other_gens(chunk_ids, gen_id)
@@ -552,6 +611,8 @@ class Repository(object):
         
         tracing.trace('putting chunk (checksum=%s)', repr(checksum))
         self.require_started_generation()
+        self.require_shared_lock()
+
         if self.prev_chunkid is None:
             self.prev_chunkid = random_chunkid()
         while True:
@@ -618,6 +679,7 @@ class Repository(object):
 
         tracing.trace('chunk_id=%s', chunk_id)
         self.require_open_client()
+        self.require_shared_lock()
         self.chunklist.remove(chunk_id)
         filename = self._chunk_filename(chunk_id)
         try:
diff --git a/obnamlib/repo_tests.py b/obnamlib/repo_tests.py
index ef28ab51..1d883a0e 100644
--- a/obnamlib/repo_tests.py
+++ b/obnamlib/repo_tests.py
@@ -37,7 +37,7 @@ class RepositoryRootNodeTests(unittest.TestCase):
                                         obnamlib.IDPATH_DEPTH,
                                         obnamlib.IDPATH_BITS,
                                         obnamlib.IDPATH_SKIP,
-                                        time.time)
+                                        time.time, 0)
         
         self.otherfs = obnamlib.LocalFS(self.tempdir)
         self.other = obnamlib.Repository(self.fs, obnamlib.DEFAULT_NODE_SIZE,
@@ -46,7 +46,7 @@ class RepositoryRootNodeTests(unittest.TestCase):
                                         obnamlib.IDPATH_DEPTH,
                                         obnamlib.IDPATH_BITS,
                                         obnamlib.IDPATH_SKIP,
-                                        time.time)
+                                        time.time, 0)
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -85,7 +85,7 @@ class RepositoryRootNodeTests(unittest.TestCase):
         
     def test_locking_root_node_twice_fails(self):
         self.repo.lock_root()
-        self.assertRaises(obnamlib.LockFail, self.repo.lock_root)
+        self.assertRaises(obnamlib.Error, self.repo.lock_root)
         
     def test_commit_releases_lock(self):
         self.repo.lock_root()
@@ -127,6 +127,23 @@ class RepositoryRootNodeTests(unittest.TestCase):
         self.repo._write_format_version(0)
         self.assertRaises(obnamlib.BadFormat, self.repo.list_clients)
 
+    def test_locks_shared(self):
+        self.repo.lock_shared()
+        self.assertTrue(self.repo.got_shared_lock)
+        
+    def test_locking_shared_twice_fails(self):
+        self.repo.lock_shared()
+        self.assertRaises(obnamlib.Error, self.repo.lock_shared)
+
+    def test_unlocks_shared(self):
+        self.repo.lock_shared()
+        self.repo.unlock_shared()
+        self.assertFalse(self.repo.got_shared_lock)
+
+    def test_unlock_shared_when_locked_by_other_fails(self):
+        self.other.lock_shared()
+        self.assertRaises(obnamlib.LockFail, self.repo.unlock_shared)
+
     def test_lock_client_fails_if_format_is_incompatible(self):
         self.repo._write_format_version(0)
         self.assertRaises(obnamlib.BadFormat, self.repo.lock_client, 'foo')
@@ -162,7 +179,7 @@ class RepositoryRootNodeTests(unittest.TestCase):
                                  obnamlib.IDPATH_DEPTH,
                                  obnamlib.IDPATH_BITS,
                                  obnamlib.IDPATH_SKIP,
-                                 time.time)
+                                 time.time, 0)
         self.assertEqual(s2.list_clients(), ['foo'])
         
     def test_adding_existing_client_fails(self):
@@ -209,13 +226,18 @@ class RepositoryRootNodeTests(unittest.TestCase):
         self.repo.lock_root()
         self.repo.add_client('foo')
         self.repo.commit_root()
+
         self.repo.lock_client('foo')
+        self.repo.lock_shared()
         self.repo.start_generation()
         self.repo.create('/', obnamlib.Metadata())
         self.repo.commit_client()
+        self.repo.commit_shared()
+
         self.repo.lock_root()
         self.repo.remove_client('foo')
         self.repo.commit_root()
+
         self.assertEqual(self.repo.list_clients(), [])
         self.assertFalse(self.fs.exists('foo'))
 
@@ -232,7 +254,7 @@ class RepositoryClientTests(unittest.TestCase):
                                         obnamlib.IDPATH_DEPTH,
                                         obnamlib.IDPATH_BITS,
                                         obnamlib.IDPATH_SKIP,
-                                        time.time)
+                                        time.time, 0)
         self.repo.lock_root()
         self.repo.add_client('client_name')
         self.repo.commit_root()
@@ -245,7 +267,7 @@ class RepositoryClientTests(unittest.TestCase):
                                          obnamlib.IDPATH_DEPTH,
                                          obnamlib.IDPATH_BITS,
                                          obnamlib.IDPATH_SKIP,
-                                         time.time)
+                                         time.time, 0)
         
         self.dir_meta = obnamlib.Metadata()
         self.dir_meta.st_mode = stat.S_IFDIR | 0777
@@ -262,7 +284,7 @@ class RepositoryClientTests(unittest.TestCase):
 
     def test_locking_client_twice_fails(self):
         self.repo.lock_client('client_name')
-        self.assertRaises(obnamlib.LockFail, self.repo.lock_client, 
+        self.assertRaises(obnamlib.Error, self.repo.lock_client, 
                           'client_name')
 
     def test_locking_nonexistent_client_fails(self):
@@ -275,22 +297,29 @@ class RepositoryClientTests(unittest.TestCase):
 
     def test_commit_client_releases_lock(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.repo.commit_client()
+        self.repo.commit_shared()
         self.assertFalse(self.repo.got_client_lock)
 
     def test_commit_does_not_mark_as_checkpoint_by_default(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.repo.start_generation()
         genid = self.repo.new_generation
         self.repo.commit_client()
+        self.repo.commit_shared()
         self.repo.open_client('client_name')
         self.assertFalse(self.repo.get_is_checkpoint(genid))
 
     def test_commit_marks_as_checkpoint_when_requested(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.repo.start_generation()
         genid = self.repo.new_generation
         self.repo.commit_client(checkpoint=True)
+        self.repo.commit_shared()
+
         self.repo.open_client('client_name')
         self.assert_(self.repo.get_is_checkpoint(genid))
 
@@ -347,8 +376,10 @@ class RepositoryClientTests(unittest.TestCase):
 
     def test_second_generation_has_different_id_from_first(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         gen = self.repo.start_generation()
         self.repo.commit_client()
+        self.repo.commit_shared()
         self.repo.lock_client('client_name')
         self.assertNotEqual(gen, self.repo.start_generation())
 
@@ -361,8 +392,11 @@ class RepositoryClientTests(unittest.TestCase):
 
     def test_commited_generation_has_start_and_end_times(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         gen = self.repo.start_generation()
         self.repo.commit_client()
+        self.repo.commit_shared()
+
         self.repo.open_client('client_name')
         start, end = self.repo.get_generation_times(gen)
         self.assertNotEqual(start, None)
@@ -371,43 +405,55 @@ class RepositoryClientTests(unittest.TestCase):
 
     def test_adding_generation_without_committing_does_not_add_it(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.repo.start_generation()
         self.repo.unlock_client()
+        self.repo.unlock_shared()
         self.repo.open_client('client_name')
         self.assertEqual(self.repo.list_generations(), [])
 
     def test_removing_generation_works(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         gen = self.repo.start_generation()
         self.repo.commit_client()
+        self.repo.commit_shared()
 
         self.repo.open_client('client_name')
         self.assertEqual(len(self.repo.list_generations()), 1)
 
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.repo.remove_generation(gen)
         self.repo.commit_client()
+        self.repo.commit_shared()
 
         self.repo.open_client('client_name')
         self.assertEqual(self.repo.list_generations(), [])
 
     def test_removing_only_second_generation_works(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         gen1 = self.repo.start_generation()
         self.repo.commit_client()
+        self.repo.commit_shared()
 
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         gen2 = self.repo.start_generation()
         chunk_id = self.repo.put_chunk('data', self.repo.checksum('data'))
         self.repo.set_file_chunks('/foo', [chunk_id])
         self.repo.commit_client()
+        self.repo.commit_shared()
 
         self.repo.open_client('client_name')
         self.assertEqual(len(self.repo.list_generations()), 2)
 
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.repo.remove_generation(gen2)
         self.repo.commit_client()
+        self.repo.commit_shared()
 
         self.repo.open_client('client_name')
         self.assertEqual(self.repo.list_generations(), [gen1])
@@ -421,11 +467,17 @@ class RepositoryClientTests(unittest.TestCase):
 
     def test_removing_without_committing_does_not_remove(self):
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         gen = self.repo.start_generation()
         self.repo.commit_client()
+        self.repo.commit_shared()
+
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.repo.remove_generation(gen)
         self.repo.unlock_client()
+        self.repo.unlock_shared()
+
         self.repo.open_client('client_name')
         self.assertEqual(self.repo.list_generations(), [gen])
 
@@ -524,7 +576,7 @@ class RepositoryChunkTests(unittest.TestCase):
                                         obnamlib.IDPATH_DEPTH,
                                         obnamlib.IDPATH_BITS,
                                         obnamlib.IDPATH_SKIP,
-                                        time.time)
+                                        time.time, 0)
         self.repo.lock_root()
         self.repo.add_client('client_name')
         self.repo.commit_root()
@@ -538,9 +590,11 @@ class RepositoryChunkTests(unittest.TestCase):
         self.assertNotEqual(self.repo.checksum('data'), None)
 
     def test_put_chunk_returns_id(self):
+        self.repo.lock_shared()
         self.assertNotEqual(self.repo.put_chunk('data', 'checksum'), None)
         
     def test_get_chunk_retrieves_what_put_chunk_puts(self):
+        self.repo.lock_shared()
         chunkid = self.repo.put_chunk('data', 'checksum')
         self.assertEqual(self.repo.get_chunk(chunkid), 'data')
         
@@ -548,18 +602,22 @@ class RepositoryChunkTests(unittest.TestCase):
         self.assertFalse(self.repo.chunk_exists(1234))
         
     def test_chunk_exists_after_it_is_put(self):
+        self.repo.lock_shared()
         chunkid = self.repo.put_chunk('chunk', 'checksum')
         self.assert_(self.repo.chunk_exists(chunkid))
 
     def test_removes_chunk(self):
+        self.repo.lock_shared()
         chunkid = self.repo.put_chunk('chunk', 'checksum')
         self.repo.remove_chunk(chunkid)
         self.assertFalse(self.repo.chunk_exists(chunkid))
 
     def test_silently_ignores_failure_when_removing_nonexistent_chunk(self):
+        self.repo.lock_shared()
         self.assertEqual(self.repo.remove_chunk(0), None)
         
     def test_find_chunks_finds_what_put_chunk_puts(self):
+        self.repo.lock_shared()
         checksum = self.repo.checksum('data')
         chunkid = self.repo.put_chunk('data', checksum)
         self.assertEqual(self.repo.find_chunks(checksum), [chunkid])
@@ -568,6 +626,7 @@ class RepositoryChunkTests(unittest.TestCase):
         self.assertEqual(self.repo.find_chunks('checksum'), [])
         
     def test_handles_checksum_collision(self):
+        self.repo.lock_shared()
         checksum = self.repo.checksum('data')
         chunkid1 = self.repo.put_chunk('data', checksum)
         chunkid2 = self.repo.put_chunk('data', checksum)
@@ -578,6 +637,7 @@ class RepositoryChunkTests(unittest.TestCase):
         self.assertEqual(self.repo.list_chunks(), [])
         
     def test_returns_chunks_after_they_exist(self):
+        self.repo.lock_shared()
         checksum = self.repo.checksum('data')
         chunkids = []
         for i in range(2):
@@ -597,7 +657,7 @@ class RepositoryGetSetChunksTests(unittest.TestCase):
                                         obnamlib.IDPATH_DEPTH,
                                         obnamlib.IDPATH_BITS,
                                         obnamlib.IDPATH_SKIP,
-                                        time.time)
+                                        time.time, 0)
         self.repo.lock_root()
         self.repo.add_client('client_name')
         self.repo.commit_root()
@@ -642,11 +702,12 @@ class RepositoryGenspecTests(unittest.TestCase):
                                         obnamlib.IDPATH_DEPTH,
                                         obnamlib.IDPATH_BITS,
                                         obnamlib.IDPATH_SKIP,
-                                        time.time)
+                                        time.time, 0)
         self.repo.lock_root()
         self.repo.add_client('client_name')
         self.repo.commit_root()
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
@@ -654,7 +715,9 @@ class RepositoryGenspecTests(unittest.TestCase):
     def backup(self):
         gen = self.repo.start_generation()
         self.repo.commit_client()
+        self.repo.commit_shared()
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         return gen
 
     def test_latest_raises_error_if_there_are_no_generations(self):
@@ -695,7 +758,7 @@ class RepositoryWalkTests(unittest.TestCase):
                                         obnamlib.IDPATH_DEPTH,
                                         obnamlib.IDPATH_BITS,
                                         obnamlib.IDPATH_SKIP,
-                                        time.time)
+                                        time.time, 0)
         self.repo.lock_root()
         self.repo.add_client('client_name')
         self.repo.commit_root()
@@ -707,6 +770,7 @@ class RepositoryWalkTests(unittest.TestCase):
         self.file_meta.st_mode = stat.S_IFREG | 0644
         
         self.repo.lock_client('client_name')
+        self.repo.lock_shared()
         self.gen = self.repo.start_generation()
         
         self.repo.create('/', self.dir_meta)
diff --git a/obnamlib/repo_tree.py b/obnamlib/repo_tree.py
index ad7bb23d..87b4cc81 100644
--- a/obnamlib/repo_tree.py
+++ b/obnamlib/repo_tree.py
@@ -65,9 +65,13 @@ class RepositoryTree(object):
 
     def start_changes(self):
         tracing.trace('start changes for %s', self.dirname)
-        if not self.fs.exists(self.dirname):
-            tracing.trace('create %s', self.dirname)
-            self.fs.mkdir(self.dirname)
+        need_init = (not self.fs.exists(self.dirname) or
+                     self.fs.listdir(self.dirname) == [] or
+                     self.fs.listdir(self.dirname) == ['lock'])
+        if need_init:
+            if not self.fs.exists(self.dirname):
+                tracing.trace('create %s', self.dirname)
+                self.fs.mkdir(self.dirname)
             self.repo.hooks.call('repository-toplevel-init', self.repo, 
                                  self.dirname)
         self.init_forest()
diff --git a/obnamlib/vfs_local.py b/obnamlib/vfs_local.py
index 9075e98e..00ca1f86 100644
--- a/obnamlib/vfs_local.py
+++ b/obnamlib/vfs_local.py
@@ -67,7 +67,13 @@ class LocalFS(obnamlib.VirtualFileSystem):
         if not self.isdir('.'):
             if create:
                 tracing.trace('creating %s', baseurl)
-                os.mkdir(baseurl)
+                try:
+                    os.mkdir(baseurl)
+                except OSError, e: # pragma: no cover
+                    # The directory might have been created concurrently
+                    # by someone else!
+                    if e.errno != errno.EEXIST:
+                        raise
             else:
                 raise OSError(errno.ENOENT, self.cwd)
 
@@ -87,7 +93,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
             self.write_file(lockname, "")
         except OSError, e:
             if e.errno == errno.EEXIST:
-                raise obnamlib.Error("Lock %s already exists" % lockname)
+                raise obnamlib.LockFail("Lock %s already exists" % lockname)
             else:
                 raise
 
@@ -192,6 +198,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
         tracing.trace('pathname=%s', pathname)
         tracing.trace('mode=%s', mode)
         f = LocalFSFile(self.join(pathname), mode)
+        tracing.trace('opened %s', pathname)
         try:
             flags = fcntl.fcntl(f.fileno(), fcntl.F_GETFL)
             flags |= os.O_NOATIME
@@ -199,6 +206,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
         except IOError, e: # pragma: no cover
             tracing.trace('fcntl F_SETFL failed: %s', repr(e))
             return f # ignore any problems setting flags
+        tracing.trace('returning ok')
         return f
 
     def exists(self, pathname):
@@ -245,14 +253,8 @@ class LocalFS(obnamlib.VirtualFileSystem):
         try:
             os.link(tempname, path)
         except OSError, e: # pragma: no cover
-            # sshfs does not implement link(2), so we fudge it here.
-            # FIXME: This has race conditions and should be made atomic.
-            if e.errno != errno.ENOSYS:
-                os.remove(tempname)
-                raise
-            if os.path.exists(path):
-                raise OSError(errno.EEXIST, os.strerror(errno.EEXIST), path)
-            os.rename(tempname, path)
+            os.remove(tempname)
+            raise
         os.remove(tempname)
 
     def overwrite_file(self, pathname, contents, make_backup=True):
@@ -282,6 +284,7 @@ class LocalFS(obnamlib.VirtualFileSystem):
         path = self.join(pathname)
         dirname = os.path.dirname(path)
         if not os.path.exists(dirname):
+            tracing.trace('os.makedirs(%s)' % dirname)
             os.makedirs(dirname)
 
         fd, tempname = tempfile.mkstemp(dir=dirname)
diff --git a/test-lock-files b/test-lock-files
new file mode 100755
index 00000000..7358181c
--- /dev/null
+++ b/test-lock-files
@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# Obnam test: test lock files.
+#
+# Copyright 2012  Lars Wirzenius
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+die()
+{
+    echo "$@" 1>&2
+    exit 1
+}
+
+[ "$#" = 2 ] || die "Bad usage, read source!"
+
+NCLIENTS="$1"
+COUNTER="$2"
+directory="$(mktemp -d)"
+pids="$(mktemp)"
+
+for i in $(seq "$NCLIENTS")
+do
+    ./lock-and-increment "$directory" "$COUNTER" &
+    echo "$!" >> "$pids"
+done
+
+errors=0
+for pid in $(cat "$pids")
+do
+    if ! wait "$pid"
+    then
+        echo "at least one client failed" 1>&2
+        errors=1
+    fi
+done
+
+n=$(cat "$directory/counter")
+wanted=$(expr "$COUNTER" '*' "$NCLIENTS")
+if [ "$n" != "$wanted" ]
+then
+    echo "counted to $n should be $wanted" 1>&2
+    errors=1
+fi
+
+rm -rf "$directory" "$pids"
+exit $errors
+
diff --git a/test-locking b/test-locking
index 86f043e1..c32afa71 100755
--- a/test-locking
+++ b/test-locking
@@ -23,24 +23,32 @@ die()
     exit 1
 }
 
-[ "$#" = 2 ] || die "Bad usage, read source!"
+[ "$#" = 4 ] || die "Bad usage, read source!"
 
 NCLIENTS="$1"
-repo="$2"
+NGENERATIONS="$2"
+repourl="$3"
+repopath="$4"
 pids="$(mktemp)"
 
+echo "Starting backups: $NCLIENTS clients $NGENERATIONS generations"
 for i in $(seq "$NCLIENTS")
 do
-    ./test-many-generations 1 "$repo" "client-$i" &
+    ./test-many-generations "$NGENERATIONS" "$repourl" "$repopath" \
+        "client-$i" > "client-$i.output" 2>&1 &
     echo "$!" >> "$pids"
 done
 
+echo "Waiting for clients to finish... this may take a long time"
 errors=0
 for pid in $(cat "$pids")
 do
     if ! wait "$pid"
     then
-        echo "at least one client failed" 1>&2
+        if [ "$errors" = 0 ]
+        then
+            echo "at least one client failed" 1>&2
+        fi
         errors=1
     fi
 done
diff --git a/test-many-generations b/test-many-generations
index 141f3995..e680587b 100755
--- a/test-many-generations
+++ b/test-many-generations
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Obnam test: backup and verify many generations of data.
 #
@@ -17,17 +17,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+set -x
+
 die()
 {
     echo "$@" 1>&2
     exit 1
 }
 
-[ "$#" = 3 ] || die "Bad usage, read source!"
+[ "$#" = 4 ] || die "Bad usage, read source!"
 
 N="$1"
-repo="$2"
-client="$3"
+repourl="$2"
+repopath="$3"
+client="$4"
 root="$(mktemp -d)"
 amount="1k"
 
@@ -36,16 +39,33 @@ cat <<EOF > "$conf"
 [config]
 client-name = $client
 quiet = yes
+log = $client.log
+trace = obnamlib
+repository = $repourl
+root = $root
 EOF
 
 seq "$N" |
 while read gen
 do
-    echo "$gen"
-    genbackupdata --quiet --create="$amount" "$root"
-    ./verification-test backup "$repo" "$root" "$conf"
+    genbackupdata --quiet --create="$amount" "$root" --seed="$RANDOM"
+    find "$root" -exec touch --date="1970-01-01 00:00:$gen" '{}' ';'
+    ./verification-test backup "$repopath" "$root" "$conf"
+done
+
+while true
+do
+    ./verification-test verify "$repopath" "$root" "$conf"
+    ret="$?"
+    if [ "$ret" = 0 ]
+    then
+        break
+    elif [ "$ret" != 42 ]
+    then
+        echo "$client failed verification" 1>&2
+        exit 1
+    fi
 done
-./verification-test verify "$repo" "$root" "$conf"
 
 rm -rf "$conf" "$root"
 
diff --git a/verification-test b/verification-test
index 30956263..d645e9f5 100755
--- a/verification-test
+++ b/verification-test
@@ -49,6 +49,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+set -ex
+
 die()
 {
     echo "$@" 1>&2
@@ -60,11 +62,21 @@ backup()
     local repo="$1"
     local dir="$2"
     local conf="$3"
-    
-    ./obnam --no-default-configs --config="$conf" -r "$repo" backup "$dir"
-    local gen=$(./obnam --no-default-configs --config="$conf" -r "$repo" \
-                    genids | tail -n1)
-    summain "$dir" -r --output="$repo/summain.$gen"
+    local client=$(awk '/client-name/ { print $3 }' "$conf")
+    local genids=$(mktemp)
+
+    ./obnam --no-default-configs --config="$conf" backup
+
+    while ! ./obnam --no-default-configs --config="$conf" genids > "$genids"
+    do
+        :
+    done
+    local gen=$(tail -n1 "$genids")
+    if [ "x$gen" = "x" ]
+    then
+        die "gen is empty"
+    fi
+    summain "$dir" -r --output="$repo/$client.$gen.summain"
 }
 
 abspath()
@@ -81,28 +93,31 @@ verify()
     local dir=$(abspath "$2")
     local conf="$3"
     local tempdir="$(mktemp -d)"
+    local client=$(awk '/client-name/ { print $3 }' "$conf")
 
-    ./obnam --no-default-configs --config="$conf" -r "$repo" genids |
+    ./obnam --no-default-configs --config="$conf" genids \
+        > "$tempdir"/gens || exit 42
     while read gen
     do
-        ./obnam --no-default-configs --config="$conf" -r "$repo" \
-            restore --to="$tempdir/$gen" --generation="$gen"
+        rm -rf "$tempdir/$gen"
+        ./obnam --no-default-configs --config="$conf" \
+            restore --to="$tempdir/$gen" --generation="$gen" || \
+            exit 42
         summain "$tempdir/$gen/$dir" -r --output="$tempdir/summain.$gen"
-        if ! diff -u "$repo/summain.$gen" "$tempdir/summain.$gen"
+        if ! diff -u "$repo/$client.$gen.summain" "$tempdir/summain.$gen"
         then
             die "generation $gen failed to restore properly, see $tempdir"
         fi
         rm -rf "$tempdir/$gen" "$tempdir/summain.$gen"
-    done || exit 1
-    rmdir "$tempdir"
+    done < "$tempdir/gens"
+    rm -rf "$tempdir"
 }
 
-[ "$#" = 3 ] || [ "$#" = 4 ] || die "Bad usage, read source!"
+[ "$#" = 4 ] || die "Bad usage, read source!"
 
-case "$#" in
-    3) repo="$2"; root="$3"; conf="/dev/null" ;;
-    4) repo="$2"; root="$3"; conf="$4" ;;
-esac
+repo="$2"
+root="$3"
+conf="$4"
 
 case "$1" in
      backup)