Add green-albatross cache settings

10 files changed, 104 insertions, 9 deletions

diff --git a/obnamlib/__init__.py b/obnamlib/__init__.py
index eff91fbd..cdb98e9e 100644
--- a/obnamlib/__init__.py
+++ b/obnamlib/__init__.py
@@ -28,8 +28,10 @@ from .defaults import (
     DEFAULT_CHUNKIDS_PER_GROUP,
     DEFAULT_NAGIOS_WARN_AGE,
     DEFAULT_NAGIOS_CRIT_AGE,
-    DEFAULT_DIR_OBJECT_CACHE_BYTES,
+    DEFAULT_DIR_BAG_BYTES,
+    DEFAULT_DIR_CACHE_BYTES,
     DEFAULT_CHUNK_CACHE_BYTES,
+    DEFAULT_CHUNK_BAG_BYTES,
 
     IDPATH_DEPTH,
     IDPATH_BITS,
diff --git a/obnamlib/app.py b/obnamlib/app.py
index f1da31a9..6b2e288f 100644
--- a/obnamlib/app.py
+++ b/obnamlib/app.py
@@ -255,7 +255,11 @@ class App(cliapp.Application):
             'hooks': self.hooks,
             'current_time': self.time,
             'chunk_size': self.settings['chunk-size'],
-            }
+            'chunk_cache_size': self.settings['chunk-cache-size'],
+            'chunk_bag_size': self.settings['chunk-bag-size'],
+            'dir_cache_size': self.settings['dir-cache-size'],
+            'dir_bag_size': self.settings['dir-bag-size'],
+        }
 
         if create:
             return self.repo_factory.create_repo(
diff --git a/obnamlib/defaults.py b/obnamlib/defaults.py
index 478433c8..e70754ee 100644
--- a/obnamlib/defaults.py
+++ b/obnamlib/defaults.py
@@ -25,7 +25,9 @@ DEFAULT_NAGIOS_WARN_AGE = '27h'
 DEFAULT_NAGIOS_CRIT_AGE = '8d'
 
 _MEBIBYTE = 1024**2
-DEFAULT_DIR_OBJECT_CACHE_BYTES = 256 * _MEBIBYTE
+DEFAULT_DIR_BAG_BYTES = 1 * _MEBIBYTE
+DEFAULT_DIR_CACHE_BYTES = 256 * _MEBIBYTE
+DEFAULT_CHUNK_BAG_BYTES = 1 * _MEBIBYTE
 DEFAULT_CHUNK_CACHE_BYTES = 1 * _MEBIBYTE
 
 # The following values have been determined empirically on a laptop
diff --git a/obnamlib/delegator.py b/obnamlib/delegator.py
index bd8b0a05..e35fdbbd 100644
--- a/obnamlib/delegator.py
+++ b/obnamlib/delegator.py
@@ -37,6 +37,8 @@ class RepositoryDelegator(obnamlib.RepositoryInterface):
 
         self._client_finder = ClientFinder()
         self._client_finder.set_current_time(kwargs['current_time'])
+        self._client_finder.set_dir_bag_size(kwargs['dir_bag_size'])
+        self._client_finder.set_dir_cache_size(kwargs['dir_cache_size'])
 
     def set_client_list_object(self, client_list):
         self._client_list = client_list
@@ -373,6 +375,8 @@ class ClientFinder(object):
         self._client_list = None
         self._clients = {}
         self._current_time = None
+        self._dir_bag_size = None
+        self._dir_cache_size = None
 
     def set_client_factory(self, client_factory):
         self._client_factory = client_factory
@@ -386,6 +390,12 @@ class ClientFinder(object):
     def set_current_time(self, current_time):
         self._current_time = current_time
 
+    def set_dir_bag_size(self, size):
+        self._dir_bag_size = size
+
+    def set_dir_cache_size(self, size):
+        self._dir_cache_size = size
+
     def find_client(self, client_name):
         if client_name not in self._clients:
             if client_name not in self._client_list.get_client_names():
@@ -397,6 +407,8 @@ class ClientFinder(object):
             dirname = self._client_list.get_client_dirname(client_name)
             client.set_dirname(dirname)
             client.set_current_time(self._current_time)
+            client.set_dir_cache_size(self._dir_cache_size)
+            client.set_dir_bag_size(self._dir_bag_size)
             self._clients[client_name] = client
 
         return self._clients[client_name]
diff --git a/obnamlib/fmt_ga/chunk_store.py b/obnamlib/fmt_ga/chunk_store.py
index df0ca4c8..dcb920bb 100644
--- a/obnamlib/fmt_ga/chunk_store.py
+++ b/obnamlib/fmt_ga/chunk_store.py
@@ -25,6 +25,7 @@ class GAChunkStore(object):
         self._fs = None
         self._dirname = 'chunk-store'
         self._max_chunk_size = None
+        self._chunk_cache_size = obnamlib.DEFAULT_CHUNK_CACHE_BYTES
         self._bag_store = None
         self._blob_store = None
 
@@ -35,8 +36,7 @@ class GAChunkStore(object):
         self._bag_store.set_location(fs, self._dirname)
         self._blob_store = obnamlib.BlobStore()
         self._blob_store.set_bag_store(self._bag_store)
-        self._blob_store.set_max_cache_bytes(
-            obnamlib.DEFAULT_CHUNK_CACHE_BYTES)
+        self._blob_store.set_max_cache_bytes(self._chunk_cache_size)
         if self._max_chunk_size is not None:
             self._blob_store.set_max_bag_size(self._max_chunk_size)
 
@@ -45,6 +45,11 @@ class GAChunkStore(object):
         if self._blob_store:
             self._blob_store.set_max_bag_size(max_chunk_size)
 
+    def set_chunk_cache_size(self, chunk_cache_size):
+        self._chunk_cache_size = chunk_cache_size
+        if self._blob_store:
+            self._blob_store.set_max_cache_bytes(chunk_cache_size)
+
     def put_chunk_content(self, content):
         self._fs.create_and_init_toplevel(self._dirname)
         return self._blob_store.put_blob(content)
diff --git a/obnamlib/fmt_ga/client.py b/obnamlib/fmt_ga/client.py
index d2a91946..d07bea77 100644
--- a/obnamlib/fmt_ga/client.py
+++ b/obnamlib/fmt_ga/client.py
@@ -37,6 +37,8 @@ class GAClient(object):
         self._client_keys = GAKeys()
         self._generations = GAGenerationList()
         self._data_is_loaded = False
+        self._dir_cache_size = obnamlib.DEFAULT_DIR_CACHE_BYTES
+        self._dir_bag_size = obnamlib.DEFAULT_DIR_BAG_BYTES
 
     def set_current_time(self, current_time):
         self._current_time = current_time
@@ -44,6 +46,16 @@ class GAClient(object):
     def set_fs(self, fs):
         self._fs = fs
 
+    def set_dir_bag_size(self, size):
+        self._dir_bag_size = size
+        if self._blob_store:
+            self._blob_store.set_max_bag_size(self._dir_bag_size)
+
+    def set_dir_cache_size(self, size):
+        self._dir_cache_size = size
+        if self._blob_store:
+            self._blob_store.set_max_cache_bytes(self._dir_cache_size)
+
     def set_dirname(self, dirname):
         self._dirname = dirname
 
@@ -79,9 +91,8 @@ class GAClient(object):
 
             self._blob_store = obnamlib.BlobStore()
             self._blob_store.set_bag_store(bag_store)
-            self._blob_store.set_max_bag_size(obnamlib.DEFAULT_NODE_SIZE)
-            self._blob_store.set_max_cache_bytes(
-                obnamlib.DEFAULT_DIR_OBJECT_CACHE_BYTES)
+            self._blob_store.set_max_bag_size(self._dir_bag_size)
+            self._blob_store.set_max_cache_bytes(self._dir_cache_size)
         return self._blob_store
 
     def _save_per_client_data(self):
diff --git a/obnamlib/fmt_ga/format.py b/obnamlib/fmt_ga/format.py
index 2c4796dd..15b33fdc 100644
--- a/obnamlib/fmt_ga/format.py
+++ b/obnamlib/fmt_ga/format.py
@@ -33,6 +33,8 @@ class RepositoryFormatGA(obnamlib.RepositoryDelegator):
         chunk_store = obnamlib.GAChunkStore()
         if 'chunk_size' in kwargs:  # pragma: no cover
             chunk_store.set_max_chunk_size(kwargs['chunk_size'])
+        if 'chunk_cache_size' in kwargs:  # pragma: no cover
+            chunk_store.set_chunk_cache_size(kwargs['chunk_cache_size'])
         self.set_chunk_store_object(chunk_store)
 
     def init_repo(self):
diff --git a/obnamlib/fmt_ga/format_tests.py b/obnamlib/fmt_ga/format_tests.py
index 287f99b5..4d2bfb9b 100644
--- a/obnamlib/fmt_ga/format_tests.py
+++ b/obnamlib/fmt_ga/format_tests.py
@@ -35,7 +35,9 @@ class RepositoryFormatGATests(obnamlib.RepositoryInterfaceTests):
 
         self.repo = obnamlib.RepositoryFormatGA(
             hooks=self.hooks,
-            current_time=time.time)
+            current_time=time.time,
+            dir_bag_size=1,
+            dir_cache_size=0)
         self.repo.set_fs(fs)
 
     def tearDown(self):
diff --git a/obnamlib/plugins/gaconfig_plugin.py b/obnamlib/plugins/gaconfig_plugin.py
new file mode 100644
index 00000000..251cfce7
--- /dev/null
+++ b/obnamlib/plugins/gaconfig_plugin.py
@@ -0,0 +1,54 @@
+# Copyright (C) 2015  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import obnamlib
+
+
+class FormatGreenAlbatrossConfigPlugin(obnamlib.ObnamPlugin):
+
+    def enable(self):
+        # Add settings related to FORMAT GREEN ALBATROSS.
+
+        ga_group = obnamlib.option_group['green-albatross'] = (
+            'Repository format green-albatross')
+
+        self.app.settings.bytesize(
+            ['dir-cache-size'],
+            'size of in-memory cache for DIR objects',
+            metavar='SIZE',
+            default=obnamlib.DEFAULT_DIR_CACHE_BYTES,
+            group=ga_group)
+
+        self.app.settings.bytesize(
+            ['dir-bag-size'],
+            'approximage maximum size of bags combining many DIR objects',
+            metavar='SIZE',
+            default=obnamlib.DEFAULT_DIR_BAG_BYTES,
+            group=ga_group)
+
+        self.app.settings.bytesize(
+            ['chunk-cache-size'],
+            'size of in-memory cache for file data chunk objects',
+            metavar='SIZE',
+            default=obnamlib.DEFAULT_CHUNK_CACHE_BYTES,
+            group=ga_group)
+
+        self.app.settings.bytesize(
+            ['chunk-bag-size'],
+            'approximate maximum size of bag combining many chunk objects',
+            metavar='SIZE',
+            default=obnamlib.DEFAULT_CHUNK_BAG_BYTES,
+            group=ga_group)
diff --git a/without-tests b/without-tests
index 72e5f068..1bdca346 100644
--- a/without-tests
+++ b/without-tests
@@ -27,6 +27,7 @@ obnamlib/plugins/forget_plugin.py
 obnamlib/plugins/fsck_plugin.py
 obnamlib/plugins/fuse_plugin.py
 obnamlib/plugins/__init__.py
+obnamlib/plugins/gaconfig_plugin.py
 obnamlib/plugins/list_formats_plugin.py
 obnamlib/plugins/one_file_system_plugin.py
 obnamlib/plugins/restore_plugin.py