diff options
author | Lars Wirzenius <liw@liw.fi> | 2017-08-01 18:06:20 +0300 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2017-08-01 18:06:20 +0300 |
commit | 91855c582f631320d00cc1cbaa440dbc4b45f351 (patch) | |
tree | 56e971e4edcda941a40f5330f6dc8fc30a714ef3 | |
parent | 3d1ccc53fefeab2167ce5f65cbc4faa8a4d60a94 (diff) | |
download | qvisqve-91855c582f631320d00cc1cbaa440dbc4b45f351.tar.gz |
Drop: apifw, slog (they're separate projects now)
-rw-r--r-- | apifw.yarn | 113 | ||||
-rw-r--r-- | apifw/__init__.py | 27 | ||||
-rw-r--r-- | apifw/apixface.py | 38 | ||||
-rw-r--r-- | apifw/bottleapp.py | 229 | ||||
-rw-r--r-- | apifw/http.py | 119 | ||||
-rw-r--r-- | apifw/token.py | 34 | ||||
-rw-r--r-- | apifw/token_tests.py | 74 | ||||
-rw-r--r-- | apitest.key | 51 | ||||
-rw-r--r-- | apitest.key.pub | 1 | ||||
-rw-r--r-- | apitest.py | 145 | ||||
-rwxr-xr-x | check | 5 | ||||
-rwxr-xr-x | run-apitest | 47 | ||||
-rwxr-xr-x | slog-errors | 61 | ||||
-rwxr-xr-x | slog-pretty | 33 | ||||
-rw-r--r-- | slog/__init__.py | 37 | ||||
-rw-r--r-- | slog/counter.py | 34 | ||||
-rw-r--r-- | slog/counter_tests.py | 50 | ||||
-rw-r--r-- | slog/slog.py | 278 | ||||
-rw-r--r-- | slog/slog_filter.py | 117 | ||||
-rw-r--r-- | slog/slog_filter_tests.py | 267 | ||||
-rw-r--r-- | slog/slog_tests.py | 258 |
21 files changed, 2 insertions, 2016 deletions
diff --git a/apifw.yarn b/apifw.yarn deleted file mode 100644 index 322430d..0000000 --- a/apifw.yarn +++ /dev/null @@ -1,113 +0,0 @@ ---- -title: apifw integration tests -... - - -# Introduction - -This is an integration test suite for the Python `apifw` module, using -`yarn`. It starts a little test application, `apitest.py` using -`gunicorn3` and verifies that it can do HTTP requests to it. It then -kills the test application. Very simple, but it makes sure the -interaction between `gunicorn3`, `bottle.py`, and the `apifw` module -works correctly. - -`apifw` is short for "application programming interface framework". -It's a silly name. Please suggest something better. - - -# Basic scenario - - - SCENARIO runs apitest OK - - GIVEN a running apitest using gunicorn3 - - WHEN client requests GET /version without token - THEN HTTP status code is 401 Unauthorized - AND response has header WWW-Authenticate containing "Bearer" - - WHEN client gets an authorization token with scope "no_version_scope" - AND client requests GET /version using token - THEN HTTP status code is 401 Unauthorized - AND response has header WWW-Authenticate containing "Bearer" - - WHEN client gets an authorization token with scope "uapi_version_get" - AND client requests GET /version using token - THEN HTTP status code is 200 OK - AND HTTP body is "version: 4.2" - - WHEN client gets an authorization token with scope "uapi_upload_put" - AND client uploads a fake jpg - THEN HTTP status code is 200 OK - AND HTTP body is "thank you for fake jpg" - - FINALLY stop apitest - - -# Step implementations - - IMPLEMENTS GIVEN a running apitest using gunicorn3 - # Set the "aud" field for access tokens. - export APITEST_AUD=test-audience - echo "$APITEST_AUD" > "$DATADIR/aud" - - # Set the "iss" field for access tokens. - export APITEST_ISS=test-issuer - echo "$APITEST_ISS" > "$DATADIR/iss" - - # Generate an RSA key for signing access tokens for the API. Key - # generation is disabled, to make test suite faster. Using - # pre-generated key instead. - #./generate-rsa-key "$DATADIR/signing-key" - export APITEST_PUBKEY="$(cat "$SRCDIR/apitest.key.pub")" - - # FIXME: It would be good for the test suite to pick a random free - # port. But that's not simple. - export APITEST_LOG="$DATADIR/apitest.log" - gunicorn --daemon --bind 127.0.0.1:12765 -p "$DATADIR/pid" \ - --log-file "$DATADIR/log" --log-level=debug \ - apitest:app - while ! curl -s http://127.0.0.1:12765/version > /dev/null - do - # Sleep in Debian can take a fractional second arg. - sleep 0.1 - done - - IMPLEMENTS FINALLY stop apitest - kill "$(cat "$DATADIR/pid")" - - IMPLEMENTS WHEN client requests GET /version without token - curl -sv "http://127.0.0.1:12765/version" > "$DATADIR/out" 2> "$DATADIR/err" - - IMPLEMENTS WHEN client requests GET /version using token - token="$(cat "$DATADIR/token")" - curl -sv -H "Authorization: Bearer $token" \ - "http://127.0.0.1:12765/version" > "$DATADIR/out" 2> "$DATADIR/err" - - IMPLEMENTS WHEN client uploads a fake jpg - token="$(cat "$DATADIR/token")" - curl -sv -H "Authorization: Bearer $token" \ - -H "Content-type: application/jpeg" \ - -d "fake jpg" \ - -X PUT \ - "http://127.0.0.1:12765/upload" > "$DATADIR/out" 2> "$DATADIR/err" - - IMPLEMENTS WHEN client gets an authorization token with scope "(.+)" - iss="$(cat "$DATADIR/iss")" - aud="$(cat "$DATADIR/aud")" - ./create-token "$SRCDIR/apitest.key" "$iss" "$aud" "$MATCH_1" > "$DATADIR/token" - - IMPLEMENTS THEN HTTP status code is (.+) - cat "$DATADIR/err" - tr -d '\r' < "$DATADIR/err" | - grep -Fx "< HTTP/1.1 $MATCH_1" - - IMPLEMENTS THEN HTTP body is "(.+)" - grep -Fx "$MATCH_1" "$DATADIR/out" - - IMPLEMENTS THEN response has header WWW-Authenticate containing "(.+)" - cat "$DATADIR/err" - tr -d '\r' < "$DATADIR/err" | - grep -Fix "< WWW-Authenticate: $MATCH_1" - diff --git a/apifw/__init__.py b/apifw/__init__.py deleted file mode 100644 index a246667..0000000 --- a/apifw/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from .apixface import Api -from .http import ( - HttpTransaction, - Response, - HTTP_OK, - HTTP_CREATED, - HTTP_NOT_FOUND, - HTTP_BAD_REQUEST, -) -from .token import create_token, decode_token -from .bottleapp import BottleApplication, create_bottle_application diff --git a/apifw/apixface.py b/apifw/apixface.py deleted file mode 100644 index 3178a5c..0000000 --- a/apifw/apixface.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -class Api: - - def find_missing_route(self, path): - # Return list of dicts to describe "routes" that should be - # added when client requests the missing path given as an - # argument. The dicts should look like: - # - # { - # "method": "GET", - # "path": "/foo/bar", - # "callback": foo_bar_db, - # } - # - # The "method" is the HTTP method, and defaults to "GET" if not given. - # "path" is the path component of the URL being requested. It may use - # Bottle.py patterns such as "/foo/<id>" for more powerful matching. - # (Note that this means that if Bottle gets replaced, the patterns may - # have to be reimplemented.) - # - # The list may be empty if no routes are to be added. - - return [] diff --git a/apifw/bottleapp.py b/apifw/bottleapp.py deleted file mode 100644 index b248858..0000000 --- a/apifw/bottleapp.py +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import logging -import re -import time - - -import bottle -import Crypto.PublicKey.RSA -import jwt - - -import apifw - - -class BottleLoggingPlugin(apifw.HttpTransaction): - - # This a binding between Bottle and apifw.HttpTransaction, as a - # Bottle plugin. We arrange the perform_transaction method to be - # called with the right arguments, in particular the callback - # function to call. We also provide Bottle specific methods to log - # the HTTP request and response, and to amend the response by - # adding a Date header. - - def apply(self, callback, route): - - def wrapper(*args, **kwargs): - try: - return self.perform_transaction(callback, *args, **kwargs) - except bottle.HTTPError as e: - self._log_error(e) - raise e - except BaseException as e: - self._log_error(e) - raise bottle.HTTPError(500, body=str(e)) - - return wrapper - - def amend_response(self): - rfc822 = time.strftime('%a, %d %b %Y %H:%M:%S %z') - bottle.response.set_header('Date', rfc822) - - def construct_request_log(self): - r = bottle.request - return { - 'method': r.method, - 'path': r.path, - } - - def construct_response_log(self): - r = bottle.response - return { - 'status': r.status_code, - 'text': r.body, - } - - -class BottleAuthorizationPlugin: - - route_pat = re.compile(r'<[^>]*>') - - def __init__(self): - self.pubkey = None - self.iss = None - self.aud = None - - def set_token_signing_public_key(self, pubkey): - self.pubkey = Crypto.PublicKey.RSA.importKey(pubkey) - - def set_expected_issuer(self, iss): - self.iss = iss - - def set_expected_audience(self, aud): - self.aud = aud - - def apply(self, callback, route): - - def wrapper(*args, **kwargs): - if self.is_authorized(route): - return callback(*args, **kwargs) - - self.raise_unauthorized('Something went wrong') - - return wrapper - - def is_authorized(self, route): - value = self.get_authorization_header(bottle.request) - token = self.parse_authorization_header(value) - claims = self.parse_token(token) - self.check_issuer(claims) - return self.scope_allows_route(claims['scope'], route) - - def get_authorization_header(self, request): - value = request.get_header('Authorization', '') - if not value: - self.raise_unauthorized('No Authorization header') - logging.debug('Request has Authorization header: good') - return value - - def parse_authorization_header(self, value): - words = value.split() - if len(words) != 2 or words[0].lower() != 'bearer': - self.raise_unauthorized('Authorization should be "Bearer TOKEN"') - logging.debug( - 'Request Authorization header looks like a bearer token: good') - return words[1] - - def parse_token(self, token): - try: - token = apifw.decode_token(token, self.pubkey, audience=self.aud) - logging.debug('Request Authorization token can be decoded: good') - return token - except jwt.InvalidTokenError as e: - self.raise_unauthorized(str(e)) - - def check_issuer(self, claims): - if claims['iss'] != self.iss: - self.raise_unauthorized( - 'Expected issuer %s, got %s' % (self.iss, claims['iss'])) - logging.debug('Token issuer is correct: good') - - def scope_allows_route(self, claim_scopes, route): - scopes = claim_scopes.split(' ') - route_scope = self.get_scope_for_route(route['method'], route['rule']) - if route_scope in scopes: - logging.debug( - 'Route scope %s is in scopes %r', route_scope, scopes) - return True - logging.error( - 'Route scope %s is NOT in scopes %r', route_scope, scopes) - return False - - def get_scope_for_route(self, method, rule): - scope = re.sub(self.route_pat, 'id', rule) - scope = scope.replace('/', '_') - scope = 'uapi%s_%s' % (scope, method) - return scope.lower() - - def raise_unauthorized(self, explanation): - headers = { - 'WWW-Authenticate': 'Bearer', - } - raise bottle.HTTPError(401, body=explanation, headers=headers) - - -class BottleApplication: - - # Provide the interface to bottle.Bottle that we need. - # Specifically, we set up a hook to call the - # Api.find_missing_route method when a request would otherwise - # return 404, and we add the routes it returns to Bottle so that - # they are there for this and future requests. - - def __init__(self, bottleapp, api): - self._bottleapp = bottleapp - self._bottleapp.add_hook('before_request', self._add_missing_route) - self._api = api - - def add_plugin(self, plugin): - self._bottleapp.install(plugin) - - def _add_missing_route(self): - try: - self._bottleapp.match(bottle.request.environ) - except bottle.HTTPError: - routes = self._api.find_missing_route(bottle.request.path) - if routes: - for route in routes: - callback = self._callback_with_body(route['callback']) - route_dict = { - 'method': route.get('method', 'GET'), - 'path': route['path'], - 'callback': callback, - } - self._bottleapp.route(**route_dict) - else: - raise - - def _callback_with_body(self, callback): - def wrapper(*args, **kwargs): - content_type, body = self._get_request_body() - response = callback(content_type, body, *args, **kwargs) - return bottle.HTTPResponse( - status=response['status'], body=response['body'], - headers=response['headers']) - return wrapper - - def _get_request_body(self): - content_type = bottle.request.get_header('Content-Type') - if content_type == 'application/json': - body = bottle.request.json - else: - body = bottle.request.body.read() - return content_type, body - - -def create_bottle_application(api, logger, config): - # Create a new bottle.Bottle application, set it up, and return it - # so that gunicorn can execute it from the main program. - - bottleapp = bottle.Bottle() - app = BottleApplication(bottleapp, api) - - plugin = BottleLoggingPlugin() - if logger: - plugin.set_dict_logger(logger) - app.add_plugin(plugin) - - authz = BottleAuthorizationPlugin() - authz.set_token_signing_public_key(config['token-public-key']) - authz.set_expected_issuer(config['token-issuer']) - authz.set_expected_audience(config['token-audience']) - app.add_plugin(authz) - - return bottleapp diff --git a/apifw/http.py b/apifw/http.py deleted file mode 100644 index e859c07..0000000 --- a/apifw/http.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -HTTP_OK = 200 -HTTP_CREATED = 201 -HTTP_NOT_FOUND = 404 -HTTP_BAD_REQUEST = 400 - - -class HttpTransaction: - - # This class executes HTTP transactions. It is called by the web - # framework, by some method that we don't specify. See the - # BottleLoggingPlugin to use Bottle. This class is independent - # from Bottle. The entry point is the "perform_tranaction" method, - # which is tgiven the callback to call for the HTTP request that - # has been made. This class makes the call, but also logs the - # request and response, and catches and logs any excpetion. - # Actually, the request and response get logged by a sub-class of - # this class, which is supposed to define the - # "construct_request_log" and "construct_response_log" methods. - # - # Log messages, as used by this class, are dicts, with no - # particular specified keys, although "msg_type" is added by this - # class. The request and response methods are supposed to return - # dicts that get merged with what this class provides. The merged - # result is then actually logged. - - def __init__(self): - self._logger = lambda log, **kwargs: None - - def construct_request_log(self): - # Override in subclass, as necessary. - return {} - - def construct_response_log(self): - # Override in subclass, as necessary. - return {} - - def amend_response(self): - # Make any necessary changes to the response. - # Override in subclass, as necessary. - pass - - def set_dict_logger(self, logger): - # Set function to actually do logging. - self._logger = logger - - def _log_request(self): - log = { - 'msg_type': 'http-request', - } - self._logger(self._combine_dicts(log, self.construct_request_log())) - - def _log_response(self): - log = { - 'msg_type': 'http-response', - } - self._logger(self._combine_dicts(log, self.construct_response_log())) - - def _log_error(self, exc): - log = { - 'msg_type': 'error', - 'msg_text': 'caught exception at runtime', - 'exception': str(exc), - } - self._logger(self._combine_dicts(log), stack_info=True) - - def _combine_dicts(self, *dicts): - log = {} - for d in dicts: - log.update(d) - return log - - def perform_transaction(self, callback, *args, **kwargs): - try: - self._log_request() - data = callback(*args, **kwargs) - self.amend_response() - self._log_response() - return data - except SystemExit: - # If we're exiting, we exit. No need to log an error. - raise - except BaseException as e: - # Everything else results in an error logged. - self._log_error(e) - raise - - -class Response: - - def __init__(self, values): - self._dict = {} - self._keys = ['status', 'headers', 'body'] - for key in self._keys: - self[key] = '' - for key, value in values.items(): - self[key] = value - - def __setitem__(self, key, value): - assert key in self._keys - self._dict[key] = value - - def __getitem__(self, key): - return self._dict[key] diff --git a/apifw/token.py b/apifw/token.py deleted file mode 100644 index 9e3df8e..0000000 --- a/apifw/token.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import jwt - - -JWT_SIGNING_ALGORITHM = 'RS512' - - -def create_token(claims, signing_key): - return jwt.encode( - claims, - signing_key.exportKey('PEM'), - algorithm=JWT_SIGNING_ALGORITHM) - - -def decode_token(token, key, audience): - return jwt.decode( - token, - key=key.exportKey('OpenSSH'), - audience=audience) diff --git a/apifw/token_tests.py b/apifw/token_tests.py deleted file mode 100644 index 12871c0..0000000 --- a/apifw/token_tests.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import time -import unittest - -import Crypto.PublicKey.RSA -import jwt - -import apifw - - -# We generate keys here, once, so we only need to do it once. - -token_signing_key = Crypto.PublicKey.RSA.generate(1024) -wrong_signing_key = Crypto.PublicKey.RSA.generate(1024) - - -class TokenTests(unittest.TestCase): - - def make_claimes(self, now): - return { - 'iss': 'https://idp.example.com', - 'sub': 'subject-uuid', - 'aud': 'audience-uuid', - 'exp': now + 3600, - 'scope': 'openid person_resource_id uapi_orgs_get uapi_orgs_post' - } - - def test_valid_token_is_valid(self): - now = int(time.time()) - claims = self.make_claimes(now) - encoded = apifw.create_token(claims, token_signing_key) - self.assertTrue(isinstance(encoded, bytes)) - self.assertEqual(len(encoded.split(b'.')), 3) - - decoded = apifw.decode_token( - encoded, - token_signing_key, - claims['aud']) - self.assertEqual(decoded, claims) - - def test_expired_token_is_invalid(self): - now = int(time.time()) - claims = self.make_claimes(now - 86400) - encoded = apifw.create_token(claims, token_signing_key) - with self.assertRaises(jwt.ExpiredSignatureError): - apifw.decode_token( - encoded, - token_signing_key, - claims['aud']) - - def test_wronly_signed_token_is_invalid(self): - now = int(time.time()) - claims = self.make_claimes(now - 86400) - encoded = apifw.create_token(claims, wrong_signing_key) - with self.assertRaises(jwt.DecodeError): - apifw.decode_token( - encoded, - token_signing_key, - claims['aud']) diff --git a/apitest.key b/apitest.key deleted file mode 100644 index e3518cd..0000000 --- a/apitest.key +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKgIBAAKCAgEA2h4IxT6ukFAzI1aqd7/zERhgvcxFwNd09BMJb6GROSznk1a4 -upjDcGqsUMjBIted+bAFbknW+lYGLs5TEkZxDhL2nU0m5iI9kxLx969L33d99tdC -OFg00hM3T2zIkEYZjBvhG1vFCiBFgKycsruk0CRjJgLDTKa+pl4uQEtEpmUnvgyu -iWOBIAustoPchrDzaF2LSMHXHTVENMPxRnZXhRj9/xcx4+yht6i9knq6cNBA2BIR -kdWVUCKbTQDsF3ZAsHT3ZmXkRLN/1txZdrAH/DNMB4FHuqk8zgv3Gm5qBjZUzbkI -fWCr92OA94Gmb4VLySHFTrsLT+BMAXBY4l9MKt56GLZ51WRIm91CBVeekQjJiaXw -k8rU2ArjTejB6CdStXzdfEmCo9YUdHINHqjXJx6YcftZQOHNpiDTCjaCGh+4UiKz -Yr4loWjzuD9oKDA4PEXgrFFIvtAAGeVWO9S+kes59UPEHl9f9heFlPgHmHeuBwY+ -GfNPDYAUtPcyQDVSfJPrqf5LsfHcLBwuyLTRjUAJGKlcPWWBlGDalJMwVK8R7WUX -B/JORdn5OUuckFiqD+TeqZC1uF2H10xXBSf6WYNFUES8YKMMfdD+xFub8Iu1kJ30 -jXUVtVjFRc4MKRaUv+6eMDLnA/T9+aKURt+k2OTdkwcvhmH1CEAVV2ew1akCAwEA -AQKCAgEAxisZftOnWBh1jbrU4D22Qibq5iYsjbtzV7ngDds2DUNeFsBoz6exkXZp -nm/3AYfy0IL7PCu8NO9paKcuVGFJoCbchygsmlQrq29ABe/vOFXhTR5f3L9PJjot -O20zf9kgpupBiFDFYaDWZMTvDXhskmss5cEG3aJ1fsP8s49vDNrE0+fDv7F3BL12 -qtB80Kb+TykGPhFXNwNJN8N6d7FXbOa7BkN1oYZBm5KkwevdblfXQjiQW/Y4VXlL -rheTaPGYbnmmuRMD5ONM19KVOb9PUfTtM7hiihXu16mJVStSCtjcDZj6PKdTNk3i -Q3040QPDSjbzg8duzKCVjY0cRHeew3qKLAKnyZDFqChU9mTvbZSx/+SdOcLRmBIE -Lcgopd5ZLOMVDgYukt2Yn98s8WlNL4vXxtyUFJlcT5PEwI2Ctq8I+8cY/1EKgmb4 -tyyZ2f6arfsKrlBstauwvQXy9nz6nd/lpQvph0ZvaY3QPRekkdKMTNtdk/tqXeEG -QOYvvbFA3QKJwg+pWU+3NK8maEYaxm2eRTTXQsLeJHkDlRkUAg1s7UumIiIvlv/Z -WOiQq8H34cEGnMUrFhFkoOHAFabsHH68P0VrOiMoRUtrBhekQUrvBdnzAE7gLToo -JRtnQmwTxqZM4KRozxIfAJYM+3qWdfBi/pAk1bhrsgHQRE1ZRI0CggEBAOTh08Wn -hPQ2d0w4hL0GZPP0RnCc4u/VLfoHwMw3MsXd8zbKljChfDwKDSsgoW0JnptvZENU -SyPClVRCVCpOLRuZvrA/TlavQ4kpV/lrCcoIp3P+/tH/bPJDJlobAFE1LpbxJT+1 -PaMCo1hbblTc6Hir/a5DEJ4Wwaz6XMjMQu0U9LWZquHXLUoHScVDVdCJyp54EAF5 -PCddI9subfRcbOZ4aTVWADK06T0Y0PXse87c0HG8k0J+JO/l32Y3fzgOndAaIehr -KgWiLWx4FuZ/fR95omCyKgAMvaXBy/W2wobeZzdUagp3+I5haw7pwi3QsW2cKr1B -ORtXR8WMoWtEVBcCggEBAPP1s6jy+LtL+GLFRrDzv1t2GeAaehf4sO8GQzCyFRpU -qI/nUuSQ44P2BsgoB30iWk1X3NHvvdboOTNJE8PhmY6URfIMUNzCfX6KpS3B08fn -gxToFRLtnOXXOe5Q05jkM2stM7Q3LOvr9NIeC0+Q69bTOh844LL7/K8CPgWPT8Md -fqLIc48Y5dXAW2WHW0tZlnCnRWkMfdojbX6khwt6UEvSeed3VKV5BDghmKf9Lz08 -K2xoRHButM1zYyee3Itn5gLVaGMOe+peZf6ItYDLopP0fPnEjEAjIz4He1xCcOMa -VbtaaKLHvmGlOc8xfeqTswrsZ2TX7r5YMbFjmiYjfD8CggEAJjFo3Tqu/PF5xsZH -oCNJBUxl1LTZSZwRJ//TNEChwFLhGuuDVGoeCQbEW8X+KevJA7b6zCFsyHLX6E+J -K+YPsONe5popwF1Or7yuaXhrEcOP7dNHQlOVIngCFlcbHnH5bEahKJhdyK3QBBZ9 -uruCL2DD3ChkxXyWpP7CLN+o61br3sHdugHmFMxSixBJaZsUrIzsXtKULx1jtldx -Ea26nlrJc5T+Q3fc080oUWE857ABOHl3OUlDcKSzOqNYH0qRGwDBV79KK9Z8LfV8 -HMp5Xp81cV1JlOiLXPWRy1bL7yV9o8X6S/TpDRlEfCCVvn9snBXLK/mORfmyiEyH -QxcL0QKCAQEA2dyLys7grXKUqLAABzpFo0n+pZE/g3TFnU98ZVpDWjZMKeassg1q -AIiPWePVfDxXZEaYnqp4YBkWK+SQ0BcB6MAlDplNBThylbT++bPkitsGxn7TgwnL -Wb4wr9BihmEUQhwl1kSHy0/2XEYUV8PVuQz9FLDYiT5bU3avKIvo8Re/5WMZP6s+ -ZPrZI/wS3WFt5cCbTcqoAUwuFjCboPZkCrI1xy1b3EIMMIxgJXUG4KqBJNigdb+H -mwn2fIVz8tKgJ9uo0v02UABpGTvAyvoPgA2QJgUOMqCuclCAK83xvf3gneWJGAVE -0TKaQ5uxFPE9rP2dAAON41IjXoTSPkjmRQKCAQEAta44u5SJyJ1A0WGQa8BJ/sSZ -HCBcuV+GsKW6l+2NHAqQqJ3ouPNC/jnoGgKtaAfSpIJLKahO6WDupQ/yubnU6LA4 -jDtfW5SsfkgJEvvCKi98kftNNuVxS+cPUncaeaEXjuUNZPI8Vpb1kHKN4V4FVN0m -P0tgHeNgfNMpCqfLxcWcO0Wukn+azmxP95jARMwwAn6VK+/fHlYGmODBH7w1NZFp -p0vFqoUJqdm++rF9tUdwfwK8gtfZ2ZgnDpOe8uoWwX5EfMsxMXB6Twf6jkLDUHtj -QXXJ+wsfMv/l/8f4ScuAfb/pwmWtvI3YiTGcpkvJHweacA6fRzgmm75TKWNixQ== ------END RSA PRIVATE KEY-----
\ No newline at end of file diff --git a/apitest.key.pub b/apitest.key.pub deleted file mode 100644 index 397dd2c..0000000 --- a/apitest.key.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDaHgjFPq6QUDMjVqp3v/MRGGC9zEXA13T0EwlvoZE5LOeTVri6mMNwaqxQyMEi1535sAVuSdb6VgYuzlMSRnEOEvadTSbmIj2TEvH3r0vfd33210I4WDTSEzdPbMiQRhmMG+EbW8UKIEWArJyyu6TQJGMmAsNMpr6mXi5AS0SmZSe+DK6JY4EgC6y2g9yGsPNoXYtIwdcdNUQ0w/FGdleFGP3/FzHj7KG3qL2Serpw0EDYEhGR1ZVQIptNAOwXdkCwdPdmZeREs3/W3Fl2sAf8M0wHgUe6qTzOC/cabmoGNlTNuQh9YKv3Y4D3gaZvhUvJIcVOuwtP4EwBcFjiX0wq3noYtnnVZEib3UIFV56RCMmJpfCTytTYCuNN6MHoJ1K1fN18SYKj1hR0cg0eqNcnHphx+1lA4c2mINMKNoIaH7hSIrNiviWhaPO4P2goMDg8ReCsUUi+0AAZ5VY71L6R6zn1Q8QeX1/2F4WU+AeYd64HBj4Z808NgBS09zJANVJ8k+up/kux8dwsHC7ItNGNQAkYqVw9ZYGUYNqUkzBUrxHtZRcH8k5F2fk5S5yQWKoP5N6pkLW4XYfXTFcFJ/pZg0VQRLxgowx90P7EW5vwi7WQnfSNdRW1WMVFzgwpFpS/7p4wMucD9P35opRG36TY5N2TBy+GYfUIQBVXZ7DVqQ==
\ No newline at end of file diff --git a/apitest.py b/apitest.py deleted file mode 100644 index b3987d8..0000000 --- a/apitest.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/python3 -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -# This is a test application for apifw, used by the apifw.yarn test -# suite. It also acts as an example of how apifw can be used. - -import logging -import os - -import yaml - -import apifw - - -# apifw requires a class to define routes on demand. This interface is provided -# by the apifw.Api class. This interface consists of a single method -# find_missing_route. In addition callback methods for particular routes -# may also be present. - -class Api(apifw.Api): - - def find_missing_route(self, path): - logging.info('find_missing_route called!\n') - return [ - { - 'path': '/version', - 'callback': self.version, - }, - { - 'method': 'PUT', - 'path': '/upload', - 'callback': self.upload, - }, - ] - - def version(self, content_type, body): - return apifw.Response({ - 'status': apifw.HTTP_OK, - 'body': 'version: 4.2', - 'headers': { - 'Content-Type': 'text/plain', - }, - }) - - def upload(self, content_type, body): - return apifw.Response({ - 'status': apifw.HTTP_OK, - 'body': 'thank you for %s\n' % body.decode('ascii'), - 'headers': { - 'Content-Type': 'text/plain', - }, - }) - - -# We want logging. gunicorn provides logging, but only of its own -# stuff, and if we log something ourselves, using logging.debug and -# its sibling functions, they don't end up in the log file defined by -# gunicorn. -# -# Instead a separate logger is defined. The logging in -# apifw.HttpTransaction are based on logging *dicts*, with arbitrary -# key/value pairs. This is because Lars liked log files that are easy -# to process programmatically. To use this, we define a "dict logger" -# function, which we tell apifw to use; by default, there is no -# logging. We also configure the Python logging library to write a log -# file; the name of the log file will be gotten from the APITEST_LOG -# environment variable. -# -# Ideally, we would parse command line arguments and a configuration -# file here, instead of using environment variables. Unfortunately, -# gunicorn wants to hog all of the command line for itself. -# -# It has been concluded that the sensible way is for a web app to no -# have command line options and to read configuration file from a -# fixed location, or have the name of the configuration file given -# using an environment variable. -# -# See also <https://stackoverflow.com/questions/8495367/>. - -def dict_logger(log, stack_info=None): - logging.info('Start log entry') - for key in sorted(log.keys()): - logging.info(' %r=%r', key, log[key]) - logging.info('Endlog entry') - if stack_info: - logging.info('Traceback', exc_info=True) - - -logfile = os.environ.get('APITEST_LOG') -if logfile: - logging.basicConfig(filename=logfile, level=logging.DEBUG) - - - -# Here is some configuaration for the framework. -# -# To validate the access tokens we'll be using in the API, we need to -# tell apifw which public key to use. We get that from an environment -# variable. It should be in OpenSSH public key format. -# -# The framework also needs to know the "audience" for which a token is -# created. We get that from the environment as well. - -config = { - 'token-public-key': os.environ['APITEST_PUBKEY'], - 'token-audience': os.environ['APITEST_AUD'], - 'token-issuer': os.environ['APITEST_ISS'], -} - - -# Here is the magic part. We create an instance of our Api class, and -# create a new Bottle application, using the -# apifw.create_bottle_application function. We assign the Bottle app -# to the variable "app", and when we start this program using -# gunicorn, we tell it to use the "app" variable. -# -# gunicorn3 --bind 127.0.0.1:12765 apitest:app -# -# gunicorn and Bottle then collaborate, using mysterious, unclear, and -# undocumented magic to run a web service. We don't know, and -# hopefully don't need to care, how the magic works. - -api = Api() -app = apifw.create_bottle_application(api, dict_logger, config) - -# If we are running this program directly with Python, and not via -# gunicorn, we can use the Bottle built-in debug server, which can -# make some things easier to debug. - -if __name__ == '__main__': - print('running in debug mode') - app.run(host='127.0.0.1', port=12765) @@ -19,9 +19,8 @@ set -eu python3 -m CoverageTestRunner --ignore-missing-from=without-tests -pep8 apifw slog qvarn -pylint3 -j0 --rcfile pylint.conf apifw slog qvarn +pep8 qvarn +pylint3 -j0 --rcfile pylint.conf qvarn -yarn apifw.yarn "$@" yarn yarns/*.yarn \ -s yarns/lib.py --shell python2 --shell-arg '' --cd-datadir "$@" diff --git a/run-apitest b/run-apitest deleted file mode 100755 index 4455e33..0000000 --- a/run-apitest +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -set -eu - - -log="$1" -token="$2" - - -tmp="$(mktemp -d)" - -cleanup() -{ - rm -rf "$tmp" -} - -trap cleanup EXIT - -export APITEST_ISS=testissuer -export APITEST_AUD=aud -export APITEST_LOG="$log" - -scopes=" -uapi_version_get -uapi_upload_put -" - -./generate-rsa-key "$tmp/key" -./create-token "$tmp/key" "$APITEST_ISS" "$APITEST_AUD" "$scopes" > "$token" -export APITEST_PUBKEY="$(cat "$tmp/key.pub")" - -gunicorn --bind 127.0.0.1:12765 -p "$tmp/pid" -w1 \ - --log-file "$log" --log-level=debug apitest:app diff --git a/slog-errors b/slog-errors deleted file mode 100755 index 920e87d..0000000 --- a/slog-errors +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -# Read Qvarn structured log files, look for errors, and output all -# messages related to a failed HTTP request (status code 5xx). - - -import json -import sys - - -def parse_log_file(filename): - with open(filename) as f: - for line in f: - yield json.loads(line) - - -def find_contexts(msgs): - return set(get_context(msg) for msg in msgs) - - -def get_context(msg): - return msg['_context'], msg['_process_id'], msg['_thread_id'] - - -def is_in_context(msg, context): - return get_context(msg) == context - - -def filter_msgs(msgs, func): - for msg in msgs: - if func(msg): - yield msg - -def has_error_status(msgs): - for msg in msgs: - if msg['msg_type'] == 'error': - return True - if msg['msg_type'] == 'http-response' and msg['status'] >= 400: - return True - if '_traceback' in msg: - return True - return False - - -def show_msgs(msgs): - sys.stdout.write(json.dumps(msgs)) - sys.stdout.write('\n') - - -msgs = [] -for filename in sys.argv[1:]: - for msg in parse_log_file(filename): - msg['_filename'] = filename - msgs.append(msg) - - -contexts = find_contexts(msgs) -for context in contexts: - context_msgs = list(filter_msgs(msgs, lambda m: is_in_context(m, context))) - if has_error_status(context_msgs): - show_msgs(context_msgs) diff --git a/slog-pretty b/slog-pretty deleted file mode 100755 index 80f07fb..0000000 --- a/slog-pretty +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -import json -import sys - -import yaml - - -def text_representer(dumper, data): - if '\n' in data: - return dumper.represent_scalar( - u'tag:yaml.org,2002:str', data, style='|') - return dumper.represent_scalar( - u'tag:yaml.org,2002:str', data, style='') - -yaml.add_representer(str, text_representer) -yaml.add_representer(unicode, text_representer) - - -def dump(f): - for line in f: - obj = json.loads(line.strip()) - yaml.dump( - obj, stream=sys.stdout, indent=4, default_flow_style=False, - explicit_start=True, explicit_end=True) - - -if len(sys.argv) == 1: - dump(sys.stdin) -else: - for filename in sys.argv[1:]: - with open(filename) as f: - dump(f) diff --git a/slog/__init__.py b/slog/__init__.py deleted file mode 100644 index 45f93b0..0000000 --- a/slog/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 2017 Lars Wirzenius -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from .counter import Counter - -from .slog import ( - StructuredLog, - NullSlogWriter, - FileSlogWriter, - SyslogSlogWriter, - SlogHandler, - hijack_logging, -) - -from .slog_filter import ( - FilterAllow, - FilterDeny, - FilterFieldHasValue, - FilterFieldValueRegexp, - FilterHasField, - FilterInclude, - FilterAny, - construct_log_filter, -) diff --git a/slog/counter.py b/slog/counter.py deleted file mode 100644 index 4760fe8..0000000 --- a/slog/counter.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015, 2016 Suomen Tilaajavastuu Oy -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import _thread - - -class Counter: - - '''A threadsafe incrementing counter.''' - - def __init__(self): - self._lock = _thread.allocate_lock() - self._counter = 0 - - def get(self): - return self._counter - - def increment(self): - with self._lock: - self._counter += 1 - return self._counter diff --git a/slog/counter_tests.py b/slog/counter_tests.py deleted file mode 100644 index 2c19fe1..0000000 --- a/slog/counter_tests.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2016 QvarnLabs Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import threading -import unittest - -import slog - - -class CounterTests(unittest.TestCase): - - def test_counts_to_a_million_in_ten_threads(self): - max_count = 100000 - num_threads = 2 - counter = slog.Counter() - - threads = [] - for _ in range(num_threads): - ct = CountThread(max_count, counter) - ct.start() - threads.append(ct) - for ct in threads: - ct.join() - - self.assertEqual(counter.get(), max_count * num_threads) - - -class CountThread(threading.Thread): - - def __init__(self, n, counter): - super(CountThread, self).__init__() - self.n = n - self.counter = counter - - def run(self): - for _ in range(self.n): - self.counter.increment() diff --git a/slog/slog.py b/slog/slog.py deleted file mode 100644 index 81d2c61..0000000 --- a/slog/slog.py +++ /dev/null @@ -1,278 +0,0 @@ -# slog.py - structured logging -# -# Copyright 2016-2017 QvarnLabs Ab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import datetime -import json -import logging -import os -import time -import traceback -import _thread -import syslog - -import slog - - -class StructuredLog: - - '''A structured logging system. - - A structured log is one that can be easily parsed - programmatically. Traditional logs are free form text, usually - with a weakly enforced line structure and some minimal metadata - prepended to each file. This class produces JSON records instead. - - See the separate manual for more background and examples of how to - use this system. - - ''' - - def __init__(self): - self._msg_counter = slog.Counter() - self._context = {} - self._writers = [] - - def close(self): - for writer, _ in self._writers: - writer.close() - self._writers = [] - - def add_log_writer(self, writer, filter_rule): - self._writers.append((writer, filter_rule)) - - def set_context(self, new_context): - thread_id = self._get_thread_id() - self._context[thread_id] = new_context - - def reset_context(self): - thread_id = self._get_thread_id() - self._context[thread_id] = None - - def log(self, msg_type, **kwargs): - exc_info = kwargs.pop('exc_info', False) - - log_obj = { - 'msg_type': msg_type, - } - for key, value in kwargs.items(): - log_obj[key] = self._convert_value(value) - - self._add_extra_fields(log_obj, exc_info) - for writer, filter_rule in self._writers: - if filter_rule.allow(log_obj): - writer.write(log_obj) - - def _convert_value(self, value): - # Convert a value into an form that's safe to write. Meaning, - # it can't be binary data, and it is UTF-8 compatible, if it's - # a string of any sort. - # - # Note that we do not need to be able to parse the value back - # again, we just need to write it to a log file in a form that - # the user will understand. At least for now. - # - # We can't do this while encoding JSON, because the Python - # JSON library doesn't seem to allow us to override how - # encoding happens for types it already knows about, only for - # other types of values. - - converters = { - int: self._nop_conversion, - float: self._nop_conversion, - bool: self._nop_conversion, - str: self._nop_conversion, - type(None): self._nop_conversion, - - list: self._convert_list_value, - dict: self._convert_dict_value, - tuple: self._convert_tuple_value, - } - - value_type = type(value) - assert value_type in converters, \ - 'Unknown data type {}'.format(value_type) - func = converters[type(value)] - converted = func(value) - return converted - - def _nop_conversion(self, value): - return value - - def _convert_list_value(self, value): - return [self._convert_value(item) for item in value] - - def _convert_tuple_value(self, value): - return tuple(self._convert_value(item) for item in value) - - def _convert_dict_value(self, value): - return { - self._convert_value(key): self._convert_value(value[key]) - for key in value - } - - def _add_extra_fields(self, log_obj, stack_info): - log_obj['_msg_number'] = self._get_next_message_number() - log_obj['_timestamp'] = self._get_current_timestamp() - log_obj['_process_id'] = self._get_process_id() - log_obj['_thread_id'] = self._get_thread_id() - log_obj['_context'] = self._context.get(self._get_thread_id()) - if stack_info: - log_obj['_traceback'] = self._get_traceback() - - def _get_next_message_number(self): - return self._msg_counter.increment() - - def _get_current_timestamp(self): - return datetime.datetime.utcnow().isoformat(' ') - - def _get_process_id(self): - return os.getpid() - - def _get_thread_id(self): - return _thread.get_ident() - - def _get_traceback(self): - return traceback.format_exc() - - -class SlogWriter: # pragma: no cover - - def write(self, log_obj): - raise NotImplementedError() - - def close(self): - raise NotImplementedError() - - -class NullSlogWriter(SlogWriter): # pragma: no cover - - def write(self, log_obj): - pass - - def close(self): - pass - - -class FileSlogWriter(SlogWriter): - - def __init__(self): - self._log_filename = None - self._log_file = None - self._bytes_max = None - self._encoder = json.JSONEncoder(sort_keys=True) - - def set_max_file_size(self, bytes_max): - self._bytes_max = bytes_max - - def get_filename(self): - return self._log_filename - - def get_rotated_filename(self, now=None): - prefix, suffix = os.path.splitext(self._log_filename) - if now is None: # pragma: no cover - now = time.localtime() - else: - now = tuple(list(now) + [0]*9)[:9] - timestamp = time.strftime('%Y%m%dT%H%M%S', now) - return '{}-{}{}'.format(prefix, timestamp, suffix) - - def set_filename(self, filename): - self._log_filename = filename - self._log_file = open(filename, 'a') - - def write(self, log_obj): - if self._log_file: - self._write_message(log_obj) - if self._bytes_max is not None: - self._rotate() - - def _write_message(self, log_obj): - msg = self._encoder.encode(log_obj) - self._log_file.write(msg + '\n') - self._log_file.flush() - - def _rotate(self): - pos = self._log_file.tell() - if pos >= self._bytes_max: - self._log_file.close() - rotated = self.get_rotated_filename() - os.rename(self._log_filename, rotated) - self.set_filename(self._log_filename) - - def close(self): - self._log_file.close() - self._log_file = None - - -class SyslogSlogWriter(SlogWriter): # pragma: no cover - - def write(self, log_obj): - encoder = json.JSONEncoder(sort_keys=True) - s = encoder.encode(log_obj) - syslog.syslog(s) - - def close(self): - pass - - -class SlogHandler(logging.Handler): # pragma: no cover - - '''A handler for the logging library to capture into a slog. - - In order to capture all logging.* log messages into a structured - log, configure the logging library to use this handler. - - ''' - - def __init__(self, log): - super(SlogHandler, self).__init__() - self.log = log - - def emit(self, record): - attr_names = { - 'msg': 'msg_text', - } - - log_args = dict() - for attr in dir(record): - if not attr.startswith('_'): - value = getattr(record, attr) - if not isinstance(value, (str, int, bool, float)): - value = repr(value) - log_args[attr_names.get(attr, attr)] = value - self.log.log('logging.' + record.levelname, **log_args) - - -def hijack_logging(log, logger_names=None): # pragma: no cover - '''Hijack log messages that come via logging.* into a slog.''' - - handler = SlogHandler(log) - - for name in logger_names or []: - logger = logging.getLogger(name) - hijack_logger_handlers(logger, handler) - - logger = logging.getLogger() - hijack_logger_handlers(logger, handler) - - -def hijack_logger_handlers(logger, handler): # pragma: no cover - logger.setLevel(logging.DEBUG) - for h in logger.handlers: - logger.removeHandler(h) - logger.addHandler(handler) diff --git a/slog/slog_filter.py b/slog/slog_filter.py deleted file mode 100644 index 1115f05..0000000 --- a/slog/slog_filter.py +++ /dev/null @@ -1,117 +0,0 @@ -# slog_filter.py - structured logging filtering -# -# Copyright 2017 QvarnLabs Ab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import re - - -class FilterRule: # pragma: no cover - - def allow(self, log_obj): - raise NotImplementedError() - - -class FilterAllow(FilterRule): - - def allow(self, log_obj): - return True - - -class FilterDeny(FilterRule): - - def allow(self, log_obj): - return False - - -class FilterHasField(FilterRule): - - def __init__(self, field_name): - self._field_name = field_name - - def allow(self, log_obj): - return self._field_name in log_obj - - -class FilterFieldHasValue(FilterRule): - - def __init__(self, field, value): - self._field = field - self._value = value - - def allow(self, log_obj): - return self._field in log_obj and log_obj[self._field] == self._value - - -class FilterFieldValueRegexp(FilterRule): - - def __init__(self, field, pattern): - self._field = field - self._pattern = pattern - - def allow(self, log_obj): - if self._field not in log_obj: - return False - value = str(log_obj[self._field]) - return re.search(self._pattern, value) is not None - - -class FilterInclude(FilterRule): - - def __init__(self, rule_dict, rule): - self._rule = rule - self._include = rule_dict.get('include', True) - - def allow(self, log_obj): - allow = self._rule.allow(log_obj) - return (self._include and allow) or (not self._include and not allow) - - -class FilterAny(FilterRule): - - def __init__(self, rules): - self._rules = rules - - def allow(self, log_obj): - return any(rule.allow(log_obj) for rule in self._rules) - - -def construct_log_filter(filters): - if not filters: - raise NoFilter() - rules = [] - for spec in filters: - rule = None - if 'field' in spec: - if 'value' in spec: - rule = FilterFieldHasValue(spec['field'], spec['value']) - elif 'regexp' in spec: - rule = FilterFieldValueRegexp( - spec['field'], spec['regexp']) - elif 'field' in spec: - rule = FilterHasField(spec['field']) - else: # pragma: no cover - rule = FilterAllow() - if 'include' in spec: - rule = FilterInclude(spec, rule) - rules.append(rule) - return FilterAny(rules) - - -class NoFilter(Exception): - - def __init__(self): - super(NoFilter, self).__init__('No log filter specified') diff --git a/slog/slog_filter_tests.py b/slog/slog_filter_tests.py deleted file mode 100644 index 401d6a0..0000000 --- a/slog/slog_filter_tests.py +++ /dev/null @@ -1,267 +0,0 @@ -# slog_filter_tests.py - unit tests for structured logging tests -# -# Copyright 2017 QvarnLabs Ab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import unittest - -import slog - - -class FilterHasFieldTests(unittest.TestCase): - - def test_allows_if_field_is_there(self): - log_obj = { - 'foo': False, - } - rule = slog.FilterHasField('foo') - self.assertTrue(rule.allow(log_obj)) - - def test_doesnt_allow_if_field_is_not_there(self): - log_obj = { - 'foo': False, - } - rule = slog.FilterHasField('bar') - self.assertFalse(rule.allow(log_obj)) - - -class FilterFieldHasValueTests(unittest.TestCase): - - def test_allows_when_field_has_wanted_value(self): - log_obj = { - 'foo': 'bar', - } - rule = slog.FilterFieldHasValue('foo', 'bar') - self.assertTrue(rule.allow(log_obj)) - - def test_doesnt_allow_when_field_has_unwanted_value(self): - log_obj = { - 'foo': 'bar', - } - rule = slog.FilterFieldHasValue('foo', 'yo') - self.assertFalse(rule.allow(log_obj)) - - -class FilterFieldValueRegexpTests(unittest.TestCase): - - def test_allows_when_value_matches_regexp(self): - log_obj = { - 'foo': 'bar', - } - rule = slog.FilterFieldValueRegexp('foo', 'b.r') - self.assertTrue(rule.allow(log_obj)) - - def test_allows_when_value_matches_regexp_if_stringified(self): - log_obj = { - 'foo': 400, - } - rule = slog.FilterFieldValueRegexp('foo', '4.*') - self.assertTrue(rule.allow(log_obj)) - - def test_doesnt_allow_when_field_isnt_there(self): - log_obj = { - 'blarf': 'yo', - } - rule = slog.FilterFieldValueRegexp('foo', 'b.r') - self.assertFalse(rule.allow(log_obj)) - - def test_doesnt_allow_when_value_doesnt_match_regexp(self): - log_obj = { - 'foo': 'yo', - } - rule = slog.FilterFieldValueRegexp('foo', 'b.r') - self.assertFalse(rule.allow(log_obj)) - - -class FilterAllowTests(unittest.TestCase): - - def test_allows_always(self): - rule = slog.FilterAllow() - self.assertTrue(rule.allow(None)) - - -class FilterDenyTests(unittest.TestCase): - - def test_allows_denies(self): - rule = slog.FilterDeny() - self.assertFalse(rule.allow(None)) - - -class FilterIncludeTests(unittest.TestCase): - - def test_allows_if_rule_allows_and_no_include(self): - allow = slog.FilterAllow() - include = slog.FilterInclude({}, allow) - self.assertTrue(include.allow(None)) - - def test_denies_if_rule_denies_and_no_include(self): - deny = slog.FilterDeny() - include = slog.FilterInclude({}, deny) - self.assertFalse(include.allow(None)) - - def test_allows_if_rule_allows_and_include_is_true(self): - allow = slog.FilterAllow() - include = slog.FilterInclude({'include': True}, allow) - self.assertTrue(include.allow({})) - - def test_denies_if_rule_denies_and_include_is_true(self): - deny = slog.FilterDeny() - include = slog.FilterInclude({'include': True}, deny) - self.assertFalse(include.allow({})) - - def test_denies_if_rule_allows_and_include_is_false(self): - allow = slog.FilterAllow() - include = slog.FilterInclude({'include': False}, allow) - self.assertFalse(include.allow({})) - - def test_allows_if_rule_denies_and_include_is_false(self): - deny = slog.FilterDeny() - include = slog.FilterInclude({'include': False}, deny) - self.assertTrue(include.allow({})) - - -class FilterAnyTests(unittest.TestCase): - - def test_allows_if_any_rule_allows(self): - rules = [slog.FilterAllow()] - any_rule = slog.FilterAny(rules) - self.assertTrue(any_rule.allow(None)) - - def test_denies_if_all_rules_deny(self): - rules = [slog.FilterDeny(), slog.FilterAllow()] - any_rule = slog.FilterAny(rules) - self.assertTrue(any_rule.allow(None)) - - -class ConstructFilterTests(unittest.TestCase): - - def test_raises_error_if_no_filters(self): - filters = [] - with self.assertRaises(Exception): - slog.construct_log_filter(filters) - - def test_handles_field_wanted(self): - filters = [ - { - 'field': 'msg_type', - }, - ] - - rule = slog.construct_log_filter(filters) - - allowed = { - 'msg_type': 'info', - } - self.assertTrue(rule.allow(allowed)) - - denied = { - 'blah_blah': 'http-response', - } - self.assertFalse(rule.allow(denied)) - - def test_handles_field_value_wanted(self): - filters = [ - { - 'field': 'msg_type', - 'value': 'info', - }, - ] - - rule = slog.construct_log_filter(filters) - - allowed = { - 'msg_type': 'info', - } - self.assertTrue(rule.allow(allowed)) - - denied = { - 'msg_type': 'http-response', - } - self.assertFalse(rule.allow(denied)) - - def test_handles_regexp_match_wanted(self): - filters = [ - { - 'field': 'status', - 'regexp': '^4' - }, - ] - - rule = slog.construct_log_filter(filters) - - allowed = { - 'status': 400, - } - self.assertTrue(rule.allow(allowed)) - - denied = { - 'status': 200, - } - self.assertFalse(rule.allow(denied)) - - def test_handles_not_included(self): - filters = [ - { - 'field': 'status', - 'value': '400', - 'include': False, - }, - ] - - rule = slog.construct_log_filter(filters) - - allowed = { - 'status': '200', - } - self.assertTrue(rule.allow(allowed)) - - denied = { - 'status': '400', - } - self.assertFalse(rule.allow(denied)) - - def test_returns_compound_filter(self): - filters = [ - { - 'field': 'msg_type', - }, - { - 'field': 'msg_type', - 'value': 'debug', - }, - { - 'field': 'status', - 'regexp': '^4', - 'include': False, - }, - ] - - rule = slog.construct_log_filter(filters) - - allowed = { - 'msg_type': 'info', - } - self.assertTrue(rule.allow(allowed)) - - also_allowed = { - 'msg_type': 'debug', - } - self.assertTrue(rule.allow(also_allowed)) - - denied = { - 'status': '400', - } - self.assertFalse(rule.allow(denied)) diff --git a/slog/slog_tests.py b/slog/slog_tests.py deleted file mode 100644 index cc18f15..0000000 --- a/slog/slog_tests.py +++ /dev/null @@ -1,258 +0,0 @@ -# slog_tests.py - unit tests for structured logging -# -# Copyright 2016 QvarnLabs Ab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import glob -import json -import os -import shutil -import tempfile -import time -import unittest - -import slog - - -class StructuredLogTests(unittest.TestCase): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def create_structured_log(self): - filename = os.path.join(self.tempdir, 'slog.log') - writer = slog.FileSlogWriter() - writer.set_filename(filename) - - log = slog.StructuredLog() - log.add_log_writer(writer, slog.FilterAllow()) - return log, writer, filename - - def read_log_entries(self, writer): - filename = writer.get_filename() - with open(filename) as f: - return [json.loads(line) for line in f] - - def test_logs_in_json(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg', foo='foo', bar='bar', number=12765) - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - obj = objs[0] - self.assertEqual(obj['msg_type'], 'testmsg') - self.assertEqual(obj['foo'], 'foo') - self.assertEqual(obj['bar'], 'bar') - self.assertEqual(obj['number'], 12765) - - def test_logs_two_lines_in_json(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg1', foo='foo') - log.log('testmsg2', bar='bar') - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 2) - obj1, obj2 = objs - - self.assertEqual(obj1['msg_type'], 'testmsg1') - self.assertEqual(obj1['foo'], 'foo') - - self.assertEqual(obj2['msg_type'], 'testmsg2') - self.assertEqual(obj2['bar'], 'bar') - - def test_adds_some_extra_fields(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg') - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - obj = objs[0] - self.assertEqual(obj['msg_type'], 'testmsg') - self.assertIn('_timestamp', obj) - self.assertIn('_process_id', obj) - self.assertIn('_thread_id', obj) - self.assertEqual(obj['_context'], None) - - def test_adds_context_when_given(self): - log, writer, _ = self.create_structured_log() - log.set_context('request 123') - log.log('testmsg') - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - obj = objs[0] - self.assertEqual(obj['_context'], 'request 123') - - def test_resets_context_when_requested(self): - log, writer, _ = self.create_structured_log() - log.set_context('request 123') - log.reset_context() - log.log('testmsg') - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - obj = objs[0] - self.assertEqual(obj['_context'], None) - - def test_counts_messages(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg') - log.log('testmsg') - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 2) - self.assertEqual(objs[0]['_msg_number'], 1) - self.assertEqual(objs[1]['_msg_number'], 2) - - def test_logs_traceback(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg', exc_info=True) - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - self.assertIn('_traceback', objs[0]) - - def test_logs_unicode(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg', text='foo') - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - self.assertEqual(objs[0]['text'], 'foo') - - def test_logs_nonutf8(self): - log, writer, _ = self.create_structured_log() - notutf8 = '\x86' - log.log('blobmsg', notutf8=notutf8) - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - self.assertEqual(objs[0]['notutf8'], notutf8) - - def test_logs_list(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg', items=[1, 2, 3]) - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - self.assertEqual(objs[0]['items'], [1, 2, 3]) - - def test_logs_tuple(self): - log, writer, _ = self.create_structured_log() - log.log('testmsg', t=(1, 2, 3)) - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - # Tuples get returned as s list. - self.assertEqual(objs[0]['t'], [1, 2, 3]) - - def test_logs_dict(self): - log, writer, _ = self.create_structured_log() - dikt = { - 'foo': 'bar', - 'yo': [1, 2, 3], - } - log.log('testmsg', dikt=dikt) - log.close() - - objs = self.read_log_entries(writer) - self.assertEqual(len(objs), 1) - self.assertEqual(objs[0]['dikt'], dikt) - - def test_logs_to_two_files(self): - filename1 = os.path.join(self.tempdir, 'slog1XS') - writer1 = slog.FileSlogWriter() - writer1.set_filename(filename1) - - filename2 = os.path.join(self.tempdir, 'slog2') - writer2 = slog.FileSlogWriter() - writer2.set_filename(filename2) - - log = slog.StructuredLog() - log.add_log_writer(writer1, slog.FilterAllow()) - log.add_log_writer(writer2, slog.FilterAllow()) - - log.log('test', msg_text='hello') - objs1 = self.read_log_entries(writer1) - objs2 = self.read_log_entries(writer2) - - self.assertEqual(objs1, objs2) - - -class FileSlogWriterTests(unittest.TestCase): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.filename = os.path.join(self.tempdir, 'foo.log') - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def test_gets_initial_filename_right(self): - fw = slog.FileSlogWriter() - fw.set_filename(self.filename) - self.assertEqual(fw.get_filename(), self.filename) - - def test_gets_rotated_filename_right(self): - fw = slog.FileSlogWriter() - fw.set_filename(self.filename) - - now = time.struct_time((1969, 9, 1, 14, 30, 42, 0, 243, 0)) - self.assertEqual( - fw.get_rotated_filename(now=now), - os.path.join(self.tempdir, 'foo-19690901T143042.log') - ) - - def test_creates_file(self): - fw = slog.FileSlogWriter() - filename = os.path.join(self.tempdir, 'slog.log') - fw.set_filename(filename) - self.assertTrue(os.path.exists(filename)) - - def test_rotates_after_size_limit(self): - fw = slog.FileSlogWriter() - filename = os.path.join(self.tempdir, 'slog.log') - fw.set_filename(filename) - fw.set_max_file_size(1) - fw.write({'foo': 'bar'}) - filenames = glob.glob(self.tempdir + '/*.log') - self.assertEqual(len(filenames), 2) - self.assertTrue(filename in filenames) - - rotated_filename = [x for x in filenames if x != filename][0] - objs1 = self.load_log_objs(filename) - objs2 = self.load_log_objs(rotated_filename) - - self.assertEqual(len(objs1), 0) - self.assertEqual(len(objs2), 1) - - def load_log_objs(self, filename): - with open(filename) as f: - return [json.loads(line) for line in f] |