diff options
| author | Lars Wirzenius <liw@liw.fi> | 2016-10-18 08:19:27 +0300 |
|---|---|---|
| committer | Lars Wirzenius <liw@liw.fi> | 2016-10-18 08:19:40 +0300 |
| commit | 6c344b130485a3fed4a015409d10f6e7576c5961 (patch) | |
| tree | 592b7361bbe259d12df118c325bd1d57294c680c | |
| parent | 6e06bb8a1a10517832f645d437ffb49257169147 (diff) | |
| download | obnam-6c344b130485a3fed4a015409d10f6e7576c5961.tar.gz | |
Flush client every 1000 files after a dir
| -rw-r--r-- | obnamlib/fmt_ga/client.py | 2 | ||||
| -rw-r--r-- | obnamlib/plugins/backup_plugin.py | 9 |
2 files changed, 9 insertions, 2 deletions
diff --git a/obnamlib/fmt_ga/client.py b/obnamlib/fmt_ga/client.py index 0719a10f..f60afb59 100644 --- a/obnamlib/fmt_ga/client.py +++ b/obnamlib/fmt_ga/client.py @@ -75,7 +75,7 @@ class GAClient(object): return self._dirname def flush(self): - pass + self._save_file_metadata() def flush_ro(self): for gen in self._generations: diff --git a/obnamlib/plugins/backup_plugin.py b/obnamlib/plugins/backup_plugin.py index 97d2673f..9597f845 100644 --- a/obnamlib/plugins/backup_plugin.py +++ b/obnamlib/plugins/backup_plugin.py @@ -387,8 +387,14 @@ class BackupPlugin(obnamlib.ObnamPlugin): self.root_metadata = self.fs.lstat(absroot) + num_files = 0 + flush_threshold = 1000 + for pathname, metadata in self.find_files(absroot): logging.info('Backing up %s', pathname) + + num_files += 1 + if not self.pretend: existed = self.repo.file_exists(self.new_generation, pathname) try: @@ -406,9 +412,10 @@ class BackupPlugin(obnamlib.ObnamPlugin): if e.errno in (errno.ENOSPC, errno.EPIPE): raise - if metadata.isdir() and not self.pretend: + if metadata.isdir() and not self.pretend and num_files >= flush_threshold: self.repo.flush_client(self.client_name) self.app.dump_memory_profile('after flushing client') + num_files = 0 if self.checkpoint_manager.time_for_checkpoint(): self.make_checkpoint() |