Change: support up/download of large blobs

This is very ugly. But it solves the problem, and apifw will need to
be re-designed to support this properly.

5 files changed, 61 insertions, 14 deletions

diff --git a/apifw/__init__.py b/apifw/__init__.py
index 7c1ce5b..348cf76 100644
--- a/apifw/__init__.py
+++ b/apifw/__init__.py
@@ -17,6 +17,7 @@ from .apixface import Api
 from .http import (
     HttpTransaction,
     Response,
+    StaticFile,
     HTTP_OK,
     HTTP_CREATED,
     HTTP_UNAUTHORIZED,
diff --git a/apifw/bottleapp.py b/apifw/bottleapp.py
index 377e49e..0aa403d 100644
--- a/apifw/bottleapp.py
+++ b/apifw/bottleapp.py
@@ -231,7 +231,11 @@ class BottleApplication:
             routes = self._api.find_missing_route(bottle.request.path)
             if routes:
                 for route in routes:
-                    callback = self._callback_with_body(route['callback'])
+                    if self._big_blob_route(route):
+                        callback = self._callback_with_big_blob(
+                            route['callback'])
+                    else:
+                        callback = self._callback_with_body(route['callback'])
                     route_dict = {
                         'method': route.get('method', 'GET'),
                         'path': route['path'],
@@ -242,6 +246,9 @@ class BottleApplication:
             else:
                 raise
 
+    def _big_blob_route(self, route):
+        return route.get('big-blobs', False)
+
     def add_routes_for_resource_type(self, rt):
         routes = self._api.find_missing_route(rt.get_path())
         for route in routes:
@@ -254,6 +261,25 @@ class BottleApplication:
             self._bottleapp.route(**route_dict)
             self._authz.set_route_authorization(route)
 
+    def _callback_with_big_blob(self, callback):
+        def wrapper(*args, **kwargs):
+            kwargs['raw_uri_path'] = bottle.request.environ.get('RAW_URI', '')
+            content_type = self._get_content_type()
+            body = self._read_body
+            response = callback(content_type, body, *args, **kwargs)
+            return bottle.HTTPResponse(
+                status=response['status'], body=response['body'],
+                headers=response['headers'])
+        return wrapper
+
+    def _get_content_type(self):
+        return bottle.request.get_header('Content-Type')
+
+    def _read_body(self, max_bytes):
+        read = bottle.request.environ['wsgi.input'].read
+        for part in bottle.request._iter_body(read, max_bytes):
+            yield part
+
     def _callback_with_body(self, callback):
         def wrapper(*args, **kwargs):
             kwargs['raw_uri_path'] = bottle.request.environ.get('RAW_URI', '')
@@ -265,7 +291,7 @@ class BottleApplication:
         return wrapper
 
     def _get_request_body(self):
-        raw_body = bottle.request.body.read()
+        raw_body = b''.join(self._read_body(1024**2))
         if bottle.request.method in ('POST', 'PUT'):
             if len(raw_body) == 0:
                 raise bottle.HTTPError(
@@ -273,7 +299,7 @@ class BottleApplication:
                     body='Empty body not allowed for PUT/POST')
 
         json_type = 'application/json'
-        content_type = bottle.request.get_header('Content-Type')
+        content_type = self._get_content_type()
         if content_type != json_type:
             return content_type, raw_body
 
diff --git a/apifw/http.py b/apifw/http.py
index 7b5a8e9..7761874 100644
--- a/apifw/http.py
+++ b/apifw/http.py
@@ -14,6 +14,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import bottle
+
+
 HTTP_OK = 200
 HTTP_CREATED = 201
 HTTP_UNAUTHORIZED = 401
@@ -105,10 +108,13 @@ class HttpTransaction:
             self._counter()
             self._log_request()
             data = callback(*args, **kwargs)
+            self._logger({'data': type(data)})
             self._log_callback()
             self.amend_response()
             self._log_response()
             return data
+        except StaticFile as e:
+            return bottle.static_file(e.filename, '/')
         except SystemExit:
             # If we're exiting, we exit. No need to log an error.
             raise
@@ -118,11 +124,21 @@ class HttpTransaction:
             raise
 
 
+class StaticFile(Exception):
+
+    def __init__(self, filename):
+        super().__init__()
+        self.filename = filename
+
+    def __getitem__(self, key):
+        return None
+
+
 class Response:
 
     def __init__(self, values):
         self._dict = {}
-        self._keys = ['status', 'headers', 'body']
+        self._keys = ['status', 'headers', 'body', 'static-file']
         for key in self._keys:
             self[key] = ''
         for key, value in values.items():
diff --git a/apitest.py b/apitest.py
index d28a61b..0a55756 100644
--- a/apitest.py
+++ b/apitest.py
@@ -19,7 +19,9 @@
 
 import logging
 import os
+import tempfile
 
+import bottle
 import yaml
 
 import apifw
@@ -34,7 +36,8 @@ class Api(apifw.Api):
 
     def __init__(self):
         super().__init__()
-        self._blob = None
+        fd, self._filename = tempfile.mkstemp()
+        os.close(fd)
 
     def find_missing_route(self, path):
         logging.info('find_missing_route called!\n')
@@ -48,6 +51,7 @@ class Api(apifw.Api):
                 'method': 'PUT',
                 'path': '/upload',
                 'callback': self.upload,
+                'big-blobs': True,
             },
             {
                 'method': 'GET',
@@ -66,23 +70,22 @@ class Api(apifw.Api):
         })
 
     def upload(self, content_type, body, **kwargs):
-        self._blob = body
+        mega = 2 ** 20
+        read = bottle.request.environ['wsgi.input'].read
+        with open(self._filename, 'wb') as f:
+            for part in body(mega):
+                f.write(part)
+
         return apifw.Response({
             'status': apifw.HTTP_OK,
-            'body': 'thank you for %s\n' % body.decode('ascii'),
+            'body': 'thank you for your data\n',
             'headers': {
                 'Content-Type': 'text/plain',
             },
         })
 
     def download(self, content_type, body, **kwargs):
-        return apifw.Response({
-            'status': apifw.HTTP_OK,
-            'body': self._blob,
-            'headers': {
-                'Content-Type': 'text/plain',
-            },
-        })
+        raise apifw.StaticFile(self._filename)
 
 
 # We want logging. gunicorn provides logging, but only of its own
diff --git a/pylint.conf b/pylint.conf
index 95f7e92..9c9cfa5 100644
--- a/pylint.conf
+++ b/pylint.conf
@@ -7,6 +7,7 @@ disable=
     missing-docstring,
     no-member,
     no-self-use,
+    protected-access,
     redefined-variable-type,
     too-few-public-methods,
     too-many-public-methods,