Make backup plugin append groups of chunkids.

2 files changed, 8 insertions, 2 deletions

diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py
index bded5ef4..11b2a805 100644
--- a/obnamlib/__init__.py
+++ b/obnamlib/__init__.py
@@ -30,6 +30,7 @@ DEFAULT_NODE_SIZE = 64 * 1024
 DEFAULT_CHUNK_SIZE = 64 * 1024
 DEFAULT_UPLOAD_QUEUE_SIZE = 1024
 DEFAULT_LRU_SIZE = 10 * 1000
+DEFAULT_CHUNKIDS_PER_GROUP = 1024
 
 # Maximum identifier for clients, chunks, files, etc. This is the largest
 # unsigned 64-bit value. In various places we assume 64-bit field sizes
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index 25726a0c..a7a3982a 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -214,14 +214,19 @@ class BackupPlugin(obnamlib.ObnamPlugin):
         self.store.set_file_chunks(filename, [])
         f = self.fs.open(filename, 'r')
         chunk_size = int(self.app.config['chunk-size'])
+        chunkids = []
         while True:
             data = f.read(chunk_size)
             if not data:
                 break
-            chunkid = self.backup_file_chunk(data)
-            self.store.append_file_chunks(filename, [chunkid])
+            chunkids.append(self.backup_file_chunk(data))
+            if len(chunkids) >= obnamlib.DEFAULT_CHUNKIDS_PER_GROUP:
+                self.store.append_file_chunks(filename, chunkids)
+                chunkids = []
             self.app.hooks.call('progress-data-uploaded', len(data))
         f.close()
+        if chunkids:
+            self.store.append_file_chunks(filename, chunkids)
         
     def backup_file_chunk(self, data):
         '''Back up a chunk of data by putting it into the store.'''