summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2016-10-17 22:27:31 +0300
committerLars Wirzenius <liw@liw.fi>2016-10-18 10:21:51 +0300
commitb1967189ea3e054305977cb586825bf222a05863 (patch)
tree1c8d0c6c766fcd7d7fd738031e1321ed7761b163
parent3cd45b826d2ae699f45b57113b4c4e5dd401935d (diff)
downloadobnam-b1967189ea3e054305977cb586825bf222a05863.tar.gz
Flush client every N directories
-rw-r--r--obnamlib/plugins/backup_plugin.py15
1 files changed, 13 insertions, 2 deletions
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py
index 97d2673f..f39e0d07 100644
--- a/obnamlib/plugins/backup_plugin.py
+++ b/obnamlib/plugins/backup_plugin.py
@@ -387,8 +387,15 @@ class BackupPlugin(obnamlib.ObnamPlugin):
self.root_metadata = self.fs.lstat(absroot)
+ num_dirs = 0
+ # The following is a very approximate guess, but we have no
+ # way of being exact.
+ dir_entry_size = 1000
+ flush_threshold = obnamlib.DEFAULT_DIR_BAG_BYTES / dir_entry_size
+
for pathname, metadata in self.find_files(absroot):
logging.info('Backing up %s', pathname)
+
if not self.pretend:
existed = self.repo.file_exists(self.new_generation, pathname)
try:
@@ -407,12 +414,16 @@ class BackupPlugin(obnamlib.ObnamPlugin):
raise
if metadata.isdir() and not self.pretend:
- self.repo.flush_client(self.client_name)
- self.app.dump_memory_profile('after flushing client')
+ num_dirs += 1
+ if num_dirs >= flush_threshold:
+ self.repo.flush_client(self.client_name)
+ self.app.dump_memory_profile('after flushing client')
+ num_dirs = 0
if self.checkpoint_manager.time_for_checkpoint():
self.make_checkpoint()
self.progress.what(pathname)
+ num_dirs = 0
self.backup_parents('.')