Merge key/client management branch.

11 files changed, 231 insertions, 41 deletions

diff --git a/obnam.1 b/obnam.1
index 5cce71be..d6968509 100644
--- a/obnam.1
+++ b/obnam.1
@@ -97,12 +97,42 @@ It verifies that all clients, generations, directories, files, and
 all file contents still exists in the backup repository.
 It may take quite a long time to run.
 .IP \(bu
-.B force-lock
+.B force\-lock
 removes a lock file for a client in the repository.
 You should only force a lock if you are sure no-one is accessing that
 client's data in the repository.
 A dangling lock might happen, for example, if obnam loses its network
 connection to the backup repository.
+.IP \(bu
+.B client\-keys
+lists the encryption key associated with each client.
+.IP \(bu
+.B list\-keys
+lists the keys that can access the repository,
+and which toplevel directories each key can access.
+Some of the toplevel directories are shared between clients,
+others are specific to a client.
+.IP \(bu
+.B list\-toplevels
+is like 
+.BR list\-keys ,
+but lists toplevels and which keys can access them.
+.IP \(bu
+.B add\-key
+adds an encryption key to the repository.
+By default, they key is added only to the shared toplevel directories,
+but it can also be added to specific clients:
+list the names of the clients on the command line.
+They key is given with the
+.B \-\-keyid
+option.
+Whoever has access to the secret key corresponding to the key id
+can access the backup repository
+(the shared toplevels plus specified clients).
+.IP \(bu
+.B remove\-key
+removes a key from the shared toplevel directories,
+plus any clients specified on the command line.
 .SS "Making backups"
 When you run a backup,
 .B obnam
@@ -440,6 +470,13 @@ Encrypt data stored in the backup repository using
 .B gpg
 and the key specified with
 .IR KEYID .
+.TP
+.BR \-\-keyid =\fIKEYID
+Key identifier to be added to or removed from the repository by the
+.B add\-key
+and
+.B remove\-key
+commands.
 .\" ------------------------------------------------------------------
 .SH "EXIT STATUS"
 .B obnam
diff --git a/obnamlib/clientlist.py b/obnamlib/clientlist.py
index c25ef6f2..2ac86bea 100644
--- a/obnamlib/clientlist.py
+++ b/obnamlib/clientlist.py
@@ -28,20 +28,28 @@ class ClientList(obnamlib.RepositoryTree):
     The list maps a client name to an arbitrary (string) identifier,
     which is unique within the repository.
     
-    The list is implemented as a B-tree, with a two-part key:
-    128-bit MD5 of client name, and 64-bit unique identifier.
-    The value is the client name.
+    The list is implemented as a B-tree, with a three-part key:
+    128-bit MD5 of client name, 64-bit unique identifier, and subkey
+    identifier. The value depends on the subkey: it's either the
+    client's full name, or the public key identifier the client
+    uses to encrypt their backups.
     
     The client's identifier is a random, unique 64-bit integer.
     
     '''
+    
+    # subkey values
+    CLIENT_NAME = 0
+    KEYID = 1
+    SUBKEY_MAX = 255
 
     def __init__(self, fs, node_size, upload_queue_size, lru_size, hooks):
         self.hash_len = len(self.hashfunc(''))
-        self.fmt = '!%dsQ' % self.hash_len
-        self.key_bytes = len(self.key('', 0))
-        self.minkey = self.hashkey('\x00' * self.hash_len, 0)
-        self.maxkey = self.hashkey('\xff' * self.hash_len, obnamlib.MAX_ID)
+        self.fmt = '!%dsQB' % self.hash_len
+        self.key_bytes = len(self.key('', 0, 0))
+        self.minkey = self.hashkey('\x00' * self.hash_len, 0, 0)
+        self.maxkey = self.hashkey('\xff' * self.hash_len, obnamlib.MAX_ID, 
+                                   self.SUBKEY_MAX)
         obnamlib.RepositoryTree.__init__(self, fs, 'clientlist', 
                                          self.key_bytes, node_size, 
                                          upload_queue_size, lru_size, hooks)
@@ -50,12 +58,12 @@ class ClientList(obnamlib.RepositoryTree):
     def hashfunc(self, string):
         return hashlib.new('md5', string).digest()
 
-    def hashkey(self, namehash, client_id):
-        return struct.pack(self.fmt, namehash, client_id)
+    def hashkey(self, namehash, client_id, subkey):
+        return struct.pack(self.fmt, namehash, client_id, subkey)
 
-    def key(self, client_name, client_id):
+    def key(self, client_name, client_id, subkey):
         h = self.hashfunc(client_name)
-        return self.hashkey(h, client_id)
+        return self.hashkey(h, client_id, subkey)
 
     def unkey(self, key):
         return struct.unpack(self.fmt, key)
@@ -66,16 +74,18 @@ class ClientList(obnamlib.RepositoryTree):
     def list_clients(self):
         if self.init_forest() and self.forest.trees:
             t = self.forest.trees[-1]
-            return [v for k, v in t.lookup_range(self.minkey, self.maxkey)]
+            return [v 
+                     for k, v in t.lookup_range(self.minkey, self.maxkey)
+                     if self.unkey(k)[2] == self.CLIENT_NAME]
         else:
             return []
 
     def find_client_id(self, t, client_name):
-        minkey = self.key(client_name, 0)
-        maxkey = self.key(client_name, obnamlib.MAX_ID)
+        minkey = self.key(client_name, 0, 0)
+        maxkey = self.key(client_name, obnamlib.MAX_ID, self.SUBKEY_MAX)
         for k, v in t.lookup_range(minkey, maxkey):
-            checksum, client_id = self.unkey(k)
-            if v == client_name:
+            checksum, client_id, subkey = self.unkey(k)
+            if subkey == self.CLIENT_NAME and v == client_name:
                 return client_id
         return None
 
@@ -90,16 +100,37 @@ class ClientList(obnamlib.RepositoryTree):
         if self.find_client_id(self.tree, client_name) is None:
             while True:
                 candidate_id = self.random_id()
-                key = self.key(client_name, candidate_id)
+                key = self.key(client_name, candidate_id, self.CLIENT_NAME)
                 try:
                     self.tree.lookup(key)
                 except KeyError:
                     break
-            self.tree.insert(self.key(client_name, candidate_id), client_name)
+            key = self.key(client_name, candidate_id, self.CLIENT_NAME)
+            self.tree.insert(key, client_name)
         
     def remove_client(self, client_name):
         self.start_changes()
         client_id = self.find_client_id(self.tree, client_name)
         if client_id is not None:
-            self.tree.remove(self.key(client_name, client_id))
+            key = self.key(client_name, client_id, self.CLIENT_NAME)
+            self.tree.remove(key)
+
+    def get_client_keyid(self, client_name):
+        if self.init_forest() and self.forest.trees:
+            t = self.forest.trees[-1]
+            client_id = self.find_client_id(t, client_name)
+            if client_id is not None:
+                key = self.key(client_name, client_id, self.KEYID)
+                for k, v in t.lookup_range(key, key):
+                    return v
+        return None
+        
+    def set_client_keyid(self, client_name, keyid):
+        self.start_changes()
+        client_id = self.find_client_id(self.tree, client_name)
+        key = self.key(client_name, client_id, self.KEYID)
+        if keyid is None:
+            self.tree.remove_range(key, key)
+        else:
+            self.tree.insert(key, keyid)
 
diff --git a/obnamlib/clientlist_tests.py b/obnamlib/clientlist_tests.py
index d54f46ed..b65e98ae 100644
--- a/obnamlib/clientlist_tests.py
+++ b/obnamlib/clientlist_tests.py
@@ -38,12 +38,13 @@ class ClientListTests(unittest.TestCase):
 
     def test_key_bytes_is_correct_length(self):
         self.assertEqual(self.list.key_bytes, 
-                         len(self.list.key('foo', 12765)))
+                         len(self.list.key('foo', 12765, 0)))
 
     def test_unkey_unpacks_key_correctly(self):
-        key = self.list.key('client name', 12765)
-        client_hash, client_id = self.list.unkey(key)
+        key = self.list.key('client name', 12765, 42)
+        client_hash, client_id, subkey = self.list.unkey(key)
         self.assertEqual(client_id, 12765)
+        self.assertEqual(subkey, 42)
 
     def test_reports_none_as_id_for_nonexistent_client(self):
         self.assertEqual(self.list.get_client_id('foo'), None)
@@ -57,12 +58,22 @@ class ClientListTests(unittest.TestCase):
 
     def test_added_client_is_listed(self):
         self.list.add_client('foo')
+        self.list.set_client_keyid('foo', 'cafebeef')
         self.assertEqual(self.list.list_clients(), ['foo'])
 
     def test_removed_client_has_none_id(self):
         self.list.add_client('foo')
         self.list.remove_client('foo')
         self.assertEqual(self.list.get_client_id('foo'), None)
+        
+    def test_removed_client_has_no_keys(self):
+        self.list.add_client('foo')
+        client_id = self.list.get_client_id('foo')
+        self.list.remove_client('foo')
+        minkey = self.list.key('foo', client_id, 0)
+        maxkey = self.list.key('foo', client_id, self.list.SUBKEY_MAX)
+        pairs = list(self.list.tree.lookup_range(minkey, maxkey))
+        self.assertEqual(pairs, [])
 
     def test_twice_added_client_exists_only_once(self):
         self.list.add_client('foo')
@@ -79,3 +90,18 @@ class ClientListTests(unittest.TestCase):
         self.assertNotEqual(self.list.get_client_id('bar'),
                             self.list.get_client_id('foo'))
 
+    def test_client_has_no_public_key_initially(self):
+        self.list.add_client('foo')
+        self.assertEqual(self.list.get_client_keyid('foo'), None)
+
+    def test_sets_client_keyid(self):
+        self.list.add_client('foo')
+        self.list.set_client_keyid('foo', 'cafebeef')
+        self.assertEqual(self.list.get_client_keyid('foo'), 'cafebeef')
+
+    def test_remove_client_keyid(self):
+        self.list.add_client('foo')
+        self.list.set_client_keyid('foo', 'cafebeef')
+        self.list.set_client_keyid('foo', None)
+        self.assertEqual(self.list.get_client_keyid('foo'), None)
+
diff --git a/obnamlib/encryption_tests.py b/obnamlib/encryption_tests.py
index ece3a13b..d9b34cf9 100644
--- a/obnamlib/encryption_tests.py
+++ b/obnamlib/encryption_tests.py
@@ -133,9 +133,10 @@ uWUO7gMi+AlnxbfXVCTEgw3xhg==
 class SecretKeyringTests(unittest.TestCase):
 
     def test_lists_correct_key(self):
-        keyid = '3B1802F81B321347'
+        keyid1 = '3B1802F81B321347'
+        keyid2 = 'DF3D13AA11E69900'
         seckeys = obnamlib.SecretKeyring(cat('test-gpghome/secring.gpg'))
-        self.assertEqual(seckeys.keyids(), [keyid])
+        self.assertEqual(sorted(seckeys.keyids()), sorted([keyid1, keyid2]))
 
 
 class PublicKeyEncryptionTests(unittest.TestCase):
diff --git a/obnamlib/plugins/encryption_plugin.py b/obnamlib/plugins/encryption_plugin.py
index 30753e53..240aad70 100644
--- a/obnamlib/plugins/encryption_plugin.py
+++ b/obnamlib/plugins/encryption_plugin.py
@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import logging
 import os
 
 import obnamlib
@@ -27,16 +28,27 @@ class EncryptionPlugin(obnamlib.ObnamPlugin):
         self.app.config.new_string(['encrypt-with'],
                                    'PGP key with which to encrypt data '
                                         'in the backup repository')
+        self.app.config.new_string(['keyid'],
+                                   'PGP key id to add to/remove from '
+                                        'the backup repository')
         
         hooks = [
             ('repository-toplevel-init', self.toplevel_init),
             ('repository-read-data', self.toplevel_read_data),
             ('repository-write-data', self.toplevel_write_data),
+            ('repository-add-client', self.add_client),
         ]
         for name, callback in hooks:
             self.app.hooks.add_callback(name, callback)
             
         self._pubkey = None
+        
+        self.app.register_command('client-keys', self.client_keys)
+        self.app.register_command('list-keys', self.list_keys)
+        self.app.register_command('list-toplevels', self.list_toplevels)
+        self.app.register_command('add-key', self.add_key)
+        self.app.register_command('remove-key', self.remove_key)
+        self.app.register_command('remove-client', self.remove_client)
 
     @property
     def keyid(self):
@@ -100,21 +112,80 @@ class EncryptionPlugin(obnamlib.ObnamPlugin):
     def remove_from_userkeys(self, repo, toplevel, keyid):
         userkeys = self.read_keyring(repo, toplevel)
         if keyid in userkeys:
+            logging.debug('removing key %s from %s' % (keyid, toplevel))
             userkeys.remove(keyid)
             self.write_keyring(repo, toplevel, userkeys)
-
-    def add_client(self, repo, client_public_key):
-        self.add_to_userkeys(repo, 'metadata', client_public_key)
-        self.add_to_userkeys(repo, 'clientlist', client_public_key)
-        self.add_to_userkeys(repo, 'chunks', client_public_key)
-        self.add_to_userkeys(repo, 'chunksums', client_public_key)
-        # client will add itself to the clientlist and create its own toplevel
-
-    def remove_client(self, repo, client_keyid):
-        # client may remove itself, since it has access to the symmetric keys
-        # we assume the client-specific toplevel has already been removed
-        self.remove_from_userkeys(repo, 'chunksums', client_keyid)
-        self.remove_from_userkeys(repo, 'chunks', client_keyid)
-        self.remove_from_userkeys(repo, 'clientlist', client_keyid)
-        self.remove_from_userkeys(repo, 'metadata', client_keyid)
+        else:
+            logging.debug('unable to remove key %s from %s (not there)' %
+                          (keyid, toplevel))
+
+    def add_client(self, clientlist, client_name):
+        clientlist.set_client_keyid(client_name, self.keyid)
+
+    def client_keys(self, args):
+        repo = self.app.open_repository()
+        clients = repo.list_clients()
+        for client in clients:
+            keyid = repo.clientlist.get_client_keyid(client)
+            if keyid is None:
+                keyid = 'no key'
+            print client, keyid
+
+    def _find_keys_and_toplevels(self, repo):
+        toplevels = repo.fs.listdir('.')
+        keys = dict()
+        tops = dict()
+        for toplevel in toplevels:
+            userkeys = self.read_keyring(repo, toplevel)
+            for keyid in userkeys.keyids():
+                keys[keyid] = keys.get(keyid, []) + [toplevel]
+                tops[toplevel] = tops.get(toplevel, []) + [keyid]
+        return keys, tops
+
+    def list_keys(self, args):
+        repo = self.app.open_repository()
+        keys, tops = self._find_keys_and_toplevels(repo)
+        for keyid in keys:
+            print 'key: %s' % keyid
+            for toplevel in keys[keyid]:
+                print '  %s' % toplevel
+
+    def list_toplevels(self, args):
+        repo = self.app.open_repository()
+        keys, tops = self._find_keys_and_toplevels(repo)
+        for toplevel in tops:
+            print 'toplevel: %s' % toplevel
+            for keyid in tops[toplevel]:
+                print '  %s' % keyid
+
+    _shared = ['chunklist', 'chunks', 'chunksums', 'clientlist', 'metadata']
+    
+    def _find_clientdirs(self, repo, client_names):
+        return [repo.client_dir(repo.clientlist.get_client_id(x))
+                 for x in client_names]
+
+    def add_key(self, args):
+        self.app.config.require('keyid')
+        repo = self.app.open_repository()
+        keyid = self.app.config['keyid']
+        key = obnamlib.get_public_key(keyid)
+        clients = self._find_clientdirs(repo, args)
+        for toplevel in self._shared + clients:
+            self.add_to_userkeys(repo, toplevel, key)
+
+    def remove_key(self, args):
+        self.app.config.require('keyid')
+        repo = self.app.open_repository()
+        keyid = self.app.config['keyid']
+        clients = self._find_clientdirs(repo, args)
+        for toplevel in self._shared + clients:
+            self.remove_from_userkeys(repo, toplevel, keyid)
+
+    def remove_client(self, args):
+        repo = self.app.open_repository()
+        repo.lock_root()
+        for client_name in args:
+            logging.info('removing client %s' % client_name)
+            repo.remove_client(client_name)
+        repo.commit_root()
 
diff --git a/obnamlib/repo.py b/obnamlib/repo.py
index c0423f01..5060653e 100644
--- a/obnamlib/repo.py
+++ b/obnamlib/repo.py
@@ -236,6 +236,7 @@ class Repository(object):
         self.hooks.new('repository-toplevel-init')
         self.hooks.new_filter('repository-read-data')
         self.hooks.new_filter('repository-write-data')
+        self.hooks.new('repository-add-client')
         
     def checksum(self, data):
         '''Return checksum of data.
@@ -306,6 +307,8 @@ class Repository(object):
         '''Commit changes to root node, and unlock it.'''
         for client_name in self.added_clients:
             self.clientlist.add_client(client_name)
+            self.hooks.call('repository-add-client', 
+                            self.clientlist, client_name)
         self.added_clients = []
         for client_name in self.removed_clients:
             client_id = self.clientlist.get_client_id(client_name)
diff --git a/test-encrypted-repo b/test-encrypted-repo
index f62d06db..4dc1ad82 100755
--- a/test-encrypted-repo
+++ b/test-encrypted-repo
@@ -3,7 +3,9 @@
 set -e
 
 cmd="./obnam --repository=temp.repo --log=temp.log --log-level=debug"
-cmd="$cmd --encrypt-with=1B321347"
+cmd="$cmd --client-name=yeehaa --encrypt-with=3B1802F81B321347"
+
+key2="DF3D13AA11E69900"
 
 rm -rf temp.gpghome temp.data temp.repo temp.restored temp.log
 
@@ -19,3 +21,22 @@ $cmd restore --generation latest --to temp.restored
 summain -r temp.restored/$(pwd)/temp.data > temp.restored.manifest
 diff -u temp.data.manifest temp.restored.manifest
 
+$cmd add-key --keyid $key2 yeehaa
+
+echo "client keys:"
+$cmd client-keys
+
+echo "list-keys:"
+$cmd list-keys
+
+echo "list-toplevels:"
+$cmd list-toplevels
+
+echo "remove key"
+$cmd remove-key --keyid $key2 yeehaa
+$cmd list-keys
+
+echo "remove client"
+$cmd remove-client yeehaa
+$cmd client-keys
+
diff --git a/test-gpghome/pubring.gpg b/test-gpghome/pubring.gpg
index 443501c5..92df0c78 100644
--- a/test-gpghome/pubring.gpg
+++ b/test-gpghome/pubring.gpg
diff --git a/test-gpghome/random_seed b/test-gpghome/random_seed
index ec7856e8..7b3659a4 100644
--- a/test-gpghome/random_seed
+++ b/test-gpghome/random_seed
diff --git a/test-gpghome/secring.gpg b/test-gpghome/secring.gpg
index 392a864c..bed16cd2 100644
--- a/test-gpghome/secring.gpg
+++ b/test-gpghome/secring.gpg
diff --git a/test-gpghome/trustdb.gpg b/test-gpghome/trustdb.gpg
index 866db949..9a63e13b 100644
--- a/test-gpghome/trustdb.gpg
+++ b/test-gpghome/trustdb.gpg