diff options
author | Lars Wirzenius <liw@liw.fi> | 2016-10-17 22:27:31 +0300 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2016-10-18 10:21:51 +0300 |
commit | b1967189ea3e054305977cb586825bf222a05863 (patch) | |
tree | 1c8d0c6c766fcd7d7fd738031e1321ed7761b163 | |
parent | 3cd45b826d2ae699f45b57113b4c4e5dd401935d (diff) | |
download | obnam-b1967189ea3e054305977cb586825bf222a05863.tar.gz |
Flush client every N directories
-rw-r--r-- | obnamlib/plugins/backup_plugin.py | 15 |
1 files changed, 13 insertions, 2 deletions
diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py index 97d2673f..f39e0d07 100644 --- a/obnamlib/plugins/backup_plugin.py +++ b/obnamlib/plugins/backup_plugin.py @@ -387,8 +387,15 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.root_metadata = self.fs.lstat(absroot) + num_dirs = 0 + # The following is a very approximate guess, but we have no + # way of being exact. + dir_entry_size = 1000 + flush_threshold = obnamlib.DEFAULT_DIR_BAG_BYTES / dir_entry_size + for pathname, metadata in self.find_files(absroot): logging.info('Backing up %s', pathname) + if not self.pretend: existed = self.repo.file_exists(self.new_generation, pathname) try: @@ -407,12 +414,16 @@ class BackupPlugin(obnamlib.ObnamPlugin): raise if metadata.isdir() and not self.pretend: - self.repo.flush_client(self.client_name) - self.app.dump_memory_profile('after flushing client') + num_dirs += 1 + if num_dirs >= flush_threshold: + self.repo.flush_client(self.client_name) + self.app.dump_memory_profile('after flushing client') + num_dirs = 0 if self.checkpoint_manager.time_for_checkpoint(): self.make_checkpoint() self.progress.what(pathname) + num_dirs = 0 self.backup_parents('.') |