Change: handle up/downloads of large blobs

5 files changed, 31 insertions, 18 deletions

diff --git a/NEWS b/NEWS
index 0fe199b..718ea95 100644
--- a/NEWS
+++ b/NEWS
@@ -22,6 +22,9 @@ Version 0.28+git, not yet released
 
 * "Blob service" has been renamed to "artifact store".
 
+* Artifact store can handle large artifacts now. Tested with 100
+  gigabytes.
+
 Version 0.28, released 2018-02-24
 ----------------------------------
 
diff --git a/debian/changelog b/debian/changelog
index c8f0aca..9f3a0fb 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,8 @@
 ick2 (0.28+git-1) UNRELEASED; urgency=medium
 
   * New upstream version.
+  * Depend on apifw version 0.32 or later, for dealing with large
+    blobs.
 
  -- Lars Wirzenius <liw@liw.fi>  Sat, 24 Feb 2018 20:53:00 +0200
 
diff --git a/debian/control b/debian/control
index 8b2cc60..ae4a4b1 100644
--- a/debian/control
+++ b/debian/control
@@ -10,7 +10,7 @@ Build-Depends: debhelper (>= 9~),
   python-cliapp,
   python3-cliapp,
   python3-coverage-test-runner,
-  python3-apifw,
+  python3-apifw (>= 0.32),
   python3-slog,
   python3-cryptography,
   python3-requests,
@@ -28,7 +28,7 @@ Depends: ${python3:Depends}, ${misc:Depends},
   python3,
   python3-bottle,
   python3-cliapp,
-  python3-apifw,
+  python3-apifw (>= 0.32),
   python3-slog,
   python3-cryptography,
   python3-requests,
diff --git a/ick2/blob_store.py b/ick2/blob_store.py
index 7d02cbf..343b018 100644
--- a/ick2/blob_store.py
+++ b/ick2/blob_store.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017  Lars Wirzenius
+# Copyright (C) 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -34,18 +34,27 @@ class BlobStore:
             os.makedirs(self._blobdir)
 
     def store_blob(self, name, blob):
-        filename = self._blob_filename(name)
+        filename = self.blob_filename(name)
         ick2.log.log(
             'trace', msg_text='store_blob', name=name, filename=filename)
         with open(filename, 'wb') as f:
             f.write(blob)
 
+    def store_blob_from_reader(self, name, func):
+        max_size = 1024**2
+        filename = self.blob_filename(name)
+        ick2.log.log(
+            'trace', msg_text='store_blob', name=name, filename=filename)
+        with open(filename, 'wb') as f:
+            for data in func(max_size):
+                f.write(data)
+
     def get_blob(self, name):
-        filename = self._blob_filename(name)
+        filename = self.blob_filename(name)
         ick2.log.log(
             'trace', msg_text='get_blob', name=name, filename=filename)
         with open(filename, 'rb') as f:
             return f.read()
 
-    def _blob_filename(self, name):
-        return os.path.join(self._blobdir, name)
+    def blob_filename(self, name):
+        return os.path.abspath(os.path.join(self._blobdir, name))
diff --git a/ick2/blobapi.py b/ick2/blobapi.py
index 1772a27..0969c90 100644
--- a/ick2/blobapi.py
+++ b/ick2/blobapi.py
@@ -1,4 +1,4 @@
-# Copyright 2017  Lars Wirzenius
+# Copyright 2017-2018  Lars Wirzenius
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,6 +14,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import apifw
+
+
 import ick2
 
 
@@ -35,6 +38,7 @@ class BlobAPI:
                 'method': 'PUT',
                 'path': path,
                 'callback': self.put_blob,
+                'big-blobs': True,
             },
             {
                 'method': 'GET',
@@ -46,17 +50,12 @@ class BlobAPI:
     def put_blob(self, content_type, body, blob_id, **kwargs):
         ick2.log.log(
             'info', msg_text='blob uploaded', blob_id=blob_id, kwargs=kwargs)
-        self._blobs.store_blob(blob_id, body)
+        self._blobs.store_blob_from_reader(blob_id, body)
         return ick2.OK('')
 
     def get_blob(self, content_type, body, blob_id, **kwargs):
+        filename = self._blobs.blob_filename(blob_id)
         ick2.log.log(
-            'info', msg_text='blob requested', blob_id=blob_id, kwargs=kwargs)
-        try:
-            blob = self._blobs.get_blob(blob_id)
-        except EnvironmentError as e:
-            return ick2.not_found(str(e))
-        headers = {
-            'Content-Type': 'application/octet-stream',
-        }
-        return ick2.OK(blob, headers=headers)
+            'info', msg_text='blob requested', blob_id=blob_id, kwargs=kwargs,
+            filename=filename)
+        raise apifw.StaticFile(filename)