Merge: support large blobs

7 files changed, 86 insertions, 6 deletions

diff --git a/NEWS b/NEWS
index 5bc1d24..56a8b89 100644
--- a/NEWS
+++ b/NEWS
@@ -7,6 +7,9 @@ This file summarizes changes between releases of `apifw`.
 Version 0.31+git, not yet released
 ----------------------------------
 
+* Add support for uploading (PUT) and downloading (GET) large files,
+  using bottle.py's `static_file` and some poking at bottle's
+  internals in a particularly ugly way.
 
 Version 0.31, released 2017-12-08
 ----------------------------------
diff --git a/apifw.yarn b/apifw.yarn
index a914ab9..064784b 100644
--- a/apifw.yarn
+++ b/apifw.yarn
@@ -38,7 +38,12 @@ It's a silly name. Please suggest something better.
     WHEN client gets an authorization token with scope "uapi_upload_put"
     AND client uploads a fake jpg
     THEN HTTP status code is 200 OK
-    AND HTTP body is "thank you for fake jpg"
+    AND HTTP body is "thank you for your data"
+
+    WHEN client gets an authorization token with scope "uapi_download_get"
+    AND client requests GET /download using token
+    THEN HTTP status code is 200 OK
+    AND HTTP body is "fake jpg"
 
     FINALLY stop apitest
 
@@ -87,6 +92,12 @@ It's a silly name. Please suggest something better.
     curl -sv -H "Authorization: Bearer $token" \
         "http://127.0.0.1:$port/version" > "$DATADIR/out" 2> "$DATADIR/err"
 
+    IMPLEMENTS WHEN client requests GET /download using token
+    token="$(cat "$DATADIR/token")"
+    port="$(cat "$DATADIR/port")"
+    curl -sv -H "Authorization: Bearer $token" \
+        "http://127.0.0.1:$port/download" > "$DATADIR/out" 2> "$DATADIR/err"
+
     IMPLEMENTS WHEN client uploads a fake jpg
     token="$(cat "$DATADIR/token")"
     port="$(cat "$DATADIR/port")"
diff --git a/apifw/__init__.py b/apifw/__init__.py
index 7c1ce5b..348cf76 100644
--- a/apifw/__init__.py
+++ b/apifw/__init__.py
@@ -17,6 +17,7 @@ from .apixface import Api
 from .http import (
     HttpTransaction,
     Response,
+    StaticFile,
     HTTP_OK,
     HTTP_CREATED,
     HTTP_UNAUTHORIZED,
diff --git a/apifw/bottleapp.py b/apifw/bottleapp.py
index 377e49e..0aa403d 100644
--- a/apifw/bottleapp.py
+++ b/apifw/bottleapp.py
@@ -231,7 +231,11 @@ class BottleApplication:
             routes = self._api.find_missing_route(bottle.request.path)
             if routes:
                 for route in routes:
-                    callback = self._callback_with_body(route['callback'])
+                    if self._big_blob_route(route):
+                        callback = self._callback_with_big_blob(
+                            route['callback'])
+                    else:
+                        callback = self._callback_with_body(route['callback'])
                     route_dict = {
                         'method': route.get('method', 'GET'),
                         'path': route['path'],
@@ -242,6 +246,9 @@ class BottleApplication:
             else:
                 raise
 
+    def _big_blob_route(self, route):
+        return route.get('big-blobs', False)
+
     def add_routes_for_resource_type(self, rt):
         routes = self._api.find_missing_route(rt.get_path())
         for route in routes:
@@ -254,6 +261,25 @@ class BottleApplication:
             self._bottleapp.route(**route_dict)
             self._authz.set_route_authorization(route)
 
+    def _callback_with_big_blob(self, callback):
+        def wrapper(*args, **kwargs):
+            kwargs['raw_uri_path'] = bottle.request.environ.get('RAW_URI', '')
+            content_type = self._get_content_type()
+            body = self._read_body
+            response = callback(content_type, body, *args, **kwargs)
+            return bottle.HTTPResponse(
+                status=response['status'], body=response['body'],
+                headers=response['headers'])
+        return wrapper
+
+    def _get_content_type(self):
+        return bottle.request.get_header('Content-Type')
+
+    def _read_body(self, max_bytes):
+        read = bottle.request.environ['wsgi.input'].read
+        for part in bottle.request._iter_body(read, max_bytes):
+            yield part
+
     def _callback_with_body(self, callback):
         def wrapper(*args, **kwargs):
             kwargs['raw_uri_path'] = bottle.request.environ.get('RAW_URI', '')
@@ -265,7 +291,7 @@ class BottleApplication:
         return wrapper
 
     def _get_request_body(self):
-        raw_body = bottle.request.body.read()
+        raw_body = b''.join(self._read_body(1024**2))
         if bottle.request.method in ('POST', 'PUT'):
             if len(raw_body) == 0:
                 raise bottle.HTTPError(
@@ -273,7 +299,7 @@ class BottleApplication:
                     body='Empty body not allowed for PUT/POST')
 
         json_type = 'application/json'
-        content_type = bottle.request.get_header('Content-Type')
+        content_type = self._get_content_type()
         if content_type != json_type:
             return content_type, raw_body
 
diff --git a/apifw/http.py b/apifw/http.py
index 7b5a8e9..7761874 100644
--- a/apifw/http.py
+++ b/apifw/http.py
@@ -14,6 +14,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import bottle
+
+
 HTTP_OK = 200
 HTTP_CREATED = 201
 HTTP_UNAUTHORIZED = 401
@@ -105,10 +108,13 @@ class HttpTransaction:
             self._counter()
             self._log_request()
             data = callback(*args, **kwargs)
+            self._logger({'data': type(data)})
             self._log_callback()
             self.amend_response()
             self._log_response()
             return data
+        except StaticFile as e:
+            return bottle.static_file(e.filename, '/')
         except SystemExit:
             # If we're exiting, we exit. No need to log an error.
             raise
@@ -118,11 +124,21 @@ class HttpTransaction:
             raise
 
 
+class StaticFile(Exception):
+
+    def __init__(self, filename):
+        super().__init__()
+        self.filename = filename
+
+    def __getitem__(self, key):
+        return None
+
+
 class Response:
 
     def __init__(self, values):
         self._dict = {}
-        self._keys = ['status', 'headers', 'body']
+        self._keys = ['status', 'headers', 'body', 'static-file']
         for key in self._keys:
             self[key] = ''
         for key, value in values.items():
diff --git a/apitest.py b/apitest.py
index 946e9e1..0a55756 100644
--- a/apitest.py
+++ b/apitest.py
@@ -19,7 +19,9 @@
 
 import logging
 import os
+import tempfile
 
+import bottle
 import yaml
 
 import apifw
@@ -32,6 +34,11 @@ import apifw
 
 class Api(apifw.Api):
 
+    def __init__(self):
+        super().__init__()
+        fd, self._filename = tempfile.mkstemp()
+        os.close(fd)
+
     def find_missing_route(self, path):
         logging.info('find_missing_route called!\n')
         return [
@@ -44,6 +51,12 @@ class Api(apifw.Api):
                 'method': 'PUT',
                 'path': '/upload',
                 'callback': self.upload,
+                'big-blobs': True,
+            },
+            {
+                'method': 'GET',
+                'path': '/download',
+                'callback': self.download,
             },
         ]
 
@@ -57,14 +70,23 @@ class Api(apifw.Api):
         })
 
     def upload(self, content_type, body, **kwargs):
+        mega = 2 ** 20
+        read = bottle.request.environ['wsgi.input'].read
+        with open(self._filename, 'wb') as f:
+            for part in body(mega):
+                f.write(part)
+
         return apifw.Response({
             'status': apifw.HTTP_OK,
-            'body': 'thank you for %s\n' % body.decode('ascii'),
+            'body': 'thank you for your data\n',
             'headers': {
                 'Content-Type': 'text/plain',
             },
         })
 
+    def download(self, content_type, body, **kwargs):
+        raise apifw.StaticFile(self._filename)
+
 
 # We want logging. gunicorn provides logging, but only of its own
 # stuff, and if we log something ourselves, using logging.debug and
diff --git a/pylint.conf b/pylint.conf
index 95f7e92..9c9cfa5 100644
--- a/pylint.conf
+++ b/pylint.conf
@@ -7,6 +7,7 @@ disable=
     missing-docstring,
     no-member,
     no-self-use,
+    protected-access,
     redefined-variable-type,
     too-few-public-methods,
     too-many-public-methods,