From f299215889b6acf8aff86243d50d4a54bb528021 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Sat, 24 Mar 2018 15:48:12 +0200 Subject: Change: support up/download of large blobs This is very ugly. But it solves the problem, and apifw will need to be re-designed to support this properly. --- apifw/__init__.py | 1 + apifw/bottleapp.py | 32 +++++++++++++++++++++++++++++--- apifw/http.py | 18 +++++++++++++++++- apitest.py | 23 +++++++++++++---------- pylint.conf | 1 + 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/apifw/__init__.py b/apifw/__init__.py index 7c1ce5b..348cf76 100644 --- a/apifw/__init__.py +++ b/apifw/__init__.py @@ -17,6 +17,7 @@ from .apixface import Api from .http import ( HttpTransaction, Response, + StaticFile, HTTP_OK, HTTP_CREATED, HTTP_UNAUTHORIZED, diff --git a/apifw/bottleapp.py b/apifw/bottleapp.py index 377e49e..0aa403d 100644 --- a/apifw/bottleapp.py +++ b/apifw/bottleapp.py @@ -231,7 +231,11 @@ class BottleApplication: routes = self._api.find_missing_route(bottle.request.path) if routes: for route in routes: - callback = self._callback_with_body(route['callback']) + if self._big_blob_route(route): + callback = self._callback_with_big_blob( + route['callback']) + else: + callback = self._callback_with_body(route['callback']) route_dict = { 'method': route.get('method', 'GET'), 'path': route['path'], @@ -242,6 +246,9 @@ class BottleApplication: else: raise + def _big_blob_route(self, route): + return route.get('big-blobs', False) + def add_routes_for_resource_type(self, rt): routes = self._api.find_missing_route(rt.get_path()) for route in routes: @@ -254,6 +261,25 @@ class BottleApplication: self._bottleapp.route(**route_dict) self._authz.set_route_authorization(route) + def _callback_with_big_blob(self, callback): + def wrapper(*args, **kwargs): + kwargs['raw_uri_path'] = bottle.request.environ.get('RAW_URI', '') + content_type = self._get_content_type() + body = self._read_body + response = callback(content_type, body, *args, **kwargs) + return bottle.HTTPResponse( + status=response['status'], body=response['body'], + headers=response['headers']) + return wrapper + + def _get_content_type(self): + return bottle.request.get_header('Content-Type') + + def _read_body(self, max_bytes): + read = bottle.request.environ['wsgi.input'].read + for part in bottle.request._iter_body(read, max_bytes): + yield part + def _callback_with_body(self, callback): def wrapper(*args, **kwargs): kwargs['raw_uri_path'] = bottle.request.environ.get('RAW_URI', '') @@ -265,7 +291,7 @@ class BottleApplication: return wrapper def _get_request_body(self): - raw_body = bottle.request.body.read() + raw_body = b''.join(self._read_body(1024**2)) if bottle.request.method in ('POST', 'PUT'): if len(raw_body) == 0: raise bottle.HTTPError( @@ -273,7 +299,7 @@ class BottleApplication: body='Empty body not allowed for PUT/POST') json_type = 'application/json' - content_type = bottle.request.get_header('Content-Type') + content_type = self._get_content_type() if content_type != json_type: return content_type, raw_body diff --git a/apifw/http.py b/apifw/http.py index 7b5a8e9..7761874 100644 --- a/apifw/http.py +++ b/apifw/http.py @@ -14,6 +14,9 @@ # along with this program. If not, see . +import bottle + + HTTP_OK = 200 HTTP_CREATED = 201 HTTP_UNAUTHORIZED = 401 @@ -105,10 +108,13 @@ class HttpTransaction: self._counter() self._log_request() data = callback(*args, **kwargs) + self._logger({'data': type(data)}) self._log_callback() self.amend_response() self._log_response() return data + except StaticFile as e: + return bottle.static_file(e.filename, '/') except SystemExit: # If we're exiting, we exit. No need to log an error. raise @@ -118,11 +124,21 @@ class HttpTransaction: raise +class StaticFile(Exception): + + def __init__(self, filename): + super().__init__() + self.filename = filename + + def __getitem__(self, key): + return None + + class Response: def __init__(self, values): self._dict = {} - self._keys = ['status', 'headers', 'body'] + self._keys = ['status', 'headers', 'body', 'static-file'] for key in self._keys: self[key] = '' for key, value in values.items(): diff --git a/apitest.py b/apitest.py index d28a61b..0a55756 100644 --- a/apitest.py +++ b/apitest.py @@ -19,7 +19,9 @@ import logging import os +import tempfile +import bottle import yaml import apifw @@ -34,7 +36,8 @@ class Api(apifw.Api): def __init__(self): super().__init__() - self._blob = None + fd, self._filename = tempfile.mkstemp() + os.close(fd) def find_missing_route(self, path): logging.info('find_missing_route called!\n') @@ -48,6 +51,7 @@ class Api(apifw.Api): 'method': 'PUT', 'path': '/upload', 'callback': self.upload, + 'big-blobs': True, }, { 'method': 'GET', @@ -66,23 +70,22 @@ class Api(apifw.Api): }) def upload(self, content_type, body, **kwargs): - self._blob = body + mega = 2 ** 20 + read = bottle.request.environ['wsgi.input'].read + with open(self._filename, 'wb') as f: + for part in body(mega): + f.write(part) + return apifw.Response({ 'status': apifw.HTTP_OK, - 'body': 'thank you for %s\n' % body.decode('ascii'), + 'body': 'thank you for your data\n', 'headers': { 'Content-Type': 'text/plain', }, }) def download(self, content_type, body, **kwargs): - return apifw.Response({ - 'status': apifw.HTTP_OK, - 'body': self._blob, - 'headers': { - 'Content-Type': 'text/plain', - }, - }) + raise apifw.StaticFile(self._filename) # We want logging. gunicorn provides logging, but only of its own diff --git a/pylint.conf b/pylint.conf index 95f7e92..9c9cfa5 100644 --- a/pylint.conf +++ b/pylint.conf @@ -7,6 +7,7 @@ disable= missing-docstring, no-member, no-self-use, + protected-access, redefined-variable-type, too-few-public-methods, too-many-public-methods, -- cgit v1.2.1