summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <liw@liw.fi>2017-08-01 18:06:20 +0300
committerLars Wirzenius <liw@liw.fi>2017-08-01 18:06:20 +0300
commit91855c582f631320d00cc1cbaa440dbc4b45f351 (patch)
tree56e971e4edcda941a40f5330f6dc8fc30a714ef3
parent3d1ccc53fefeab2167ce5f65cbc4faa8a4d60a94 (diff)
downloadqvisqve-91855c582f631320d00cc1cbaa440dbc4b45f351.tar.gz
Drop: apifw, slog (they're separate projects now)
-rw-r--r--apifw.yarn113
-rw-r--r--apifw/__init__.py27
-rw-r--r--apifw/apixface.py38
-rw-r--r--apifw/bottleapp.py229
-rw-r--r--apifw/http.py119
-rw-r--r--apifw/token.py34
-rw-r--r--apifw/token_tests.py74
-rw-r--r--apitest.key51
-rw-r--r--apitest.key.pub1
-rw-r--r--apitest.py145
-rwxr-xr-xcheck5
-rwxr-xr-xrun-apitest47
-rwxr-xr-xslog-errors61
-rwxr-xr-xslog-pretty33
-rw-r--r--slog/__init__.py37
-rw-r--r--slog/counter.py34
-rw-r--r--slog/counter_tests.py50
-rw-r--r--slog/slog.py278
-rw-r--r--slog/slog_filter.py117
-rw-r--r--slog/slog_filter_tests.py267
-rw-r--r--slog/slog_tests.py258
21 files changed, 2 insertions, 2016 deletions
diff --git a/apifw.yarn b/apifw.yarn
deleted file mode 100644
index 322430d..0000000
--- a/apifw.yarn
+++ /dev/null
@@ -1,113 +0,0 @@
----
-title: apifw integration tests
-...
-
-
-# Introduction
-
-This is an integration test suite for the Python `apifw` module, using
-`yarn`. It starts a little test application, `apitest.py` using
-`gunicorn3` and verifies that it can do HTTP requests to it. It then
-kills the test application. Very simple, but it makes sure the
-interaction between `gunicorn3`, `bottle.py`, and the `apifw` module
-works correctly.
-
-`apifw` is short for "application programming interface framework".
-It's a silly name. Please suggest something better.
-
-
-# Basic scenario
-
-
- SCENARIO runs apitest OK
-
- GIVEN a running apitest using gunicorn3
-
- WHEN client requests GET /version without token
- THEN HTTP status code is 401 Unauthorized
- AND response has header WWW-Authenticate containing "Bearer"
-
- WHEN client gets an authorization token with scope "no_version_scope"
- AND client requests GET /version using token
- THEN HTTP status code is 401 Unauthorized
- AND response has header WWW-Authenticate containing "Bearer"
-
- WHEN client gets an authorization token with scope "uapi_version_get"
- AND client requests GET /version using token
- THEN HTTP status code is 200 OK
- AND HTTP body is "version: 4.2"
-
- WHEN client gets an authorization token with scope "uapi_upload_put"
- AND client uploads a fake jpg
- THEN HTTP status code is 200 OK
- AND HTTP body is "thank you for fake jpg"
-
- FINALLY stop apitest
-
-
-# Step implementations
-
- IMPLEMENTS GIVEN a running apitest using gunicorn3
- # Set the "aud" field for access tokens.
- export APITEST_AUD=test-audience
- echo "$APITEST_AUD" > "$DATADIR/aud"
-
- # Set the "iss" field for access tokens.
- export APITEST_ISS=test-issuer
- echo "$APITEST_ISS" > "$DATADIR/iss"
-
- # Generate an RSA key for signing access tokens for the API. Key
- # generation is disabled, to make test suite faster. Using
- # pre-generated key instead.
- #./generate-rsa-key "$DATADIR/signing-key"
- export APITEST_PUBKEY="$(cat "$SRCDIR/apitest.key.pub")"
-
- # FIXME: It would be good for the test suite to pick a random free
- # port. But that's not simple.
- export APITEST_LOG="$DATADIR/apitest.log"
- gunicorn --daemon --bind 127.0.0.1:12765 -p "$DATADIR/pid" \
- --log-file "$DATADIR/log" --log-level=debug \
- apitest:app
- while ! curl -s http://127.0.0.1:12765/version > /dev/null
- do
- # Sleep in Debian can take a fractional second arg.
- sleep 0.1
- done
-
- IMPLEMENTS FINALLY stop apitest
- kill "$(cat "$DATADIR/pid")"
-
- IMPLEMENTS WHEN client requests GET /version without token
- curl -sv "http://127.0.0.1:12765/version" > "$DATADIR/out" 2> "$DATADIR/err"
-
- IMPLEMENTS WHEN client requests GET /version using token
- token="$(cat "$DATADIR/token")"
- curl -sv -H "Authorization: Bearer $token" \
- "http://127.0.0.1:12765/version" > "$DATADIR/out" 2> "$DATADIR/err"
-
- IMPLEMENTS WHEN client uploads a fake jpg
- token="$(cat "$DATADIR/token")"
- curl -sv -H "Authorization: Bearer $token" \
- -H "Content-type: application/jpeg" \
- -d "fake jpg" \
- -X PUT \
- "http://127.0.0.1:12765/upload" > "$DATADIR/out" 2> "$DATADIR/err"
-
- IMPLEMENTS WHEN client gets an authorization token with scope "(.+)"
- iss="$(cat "$DATADIR/iss")"
- aud="$(cat "$DATADIR/aud")"
- ./create-token "$SRCDIR/apitest.key" "$iss" "$aud" "$MATCH_1" > "$DATADIR/token"
-
- IMPLEMENTS THEN HTTP status code is (.+)
- cat "$DATADIR/err"
- tr -d '\r' < "$DATADIR/err" |
- grep -Fx "< HTTP/1.1 $MATCH_1"
-
- IMPLEMENTS THEN HTTP body is "(.+)"
- grep -Fx "$MATCH_1" "$DATADIR/out"
-
- IMPLEMENTS THEN response has header WWW-Authenticate containing "(.+)"
- cat "$DATADIR/err"
- tr -d '\r' < "$DATADIR/err" |
- grep -Fix "< WWW-Authenticate: $MATCH_1"
-
diff --git a/apifw/__init__.py b/apifw/__init__.py
deleted file mode 100644
index a246667..0000000
--- a/apifw/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-from .apixface import Api
-from .http import (
- HttpTransaction,
- Response,
- HTTP_OK,
- HTTP_CREATED,
- HTTP_NOT_FOUND,
- HTTP_BAD_REQUEST,
-)
-from .token import create_token, decode_token
-from .bottleapp import BottleApplication, create_bottle_application
diff --git a/apifw/apixface.py b/apifw/apixface.py
deleted file mode 100644
index 3178a5c..0000000
--- a/apifw/apixface.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-class Api:
-
- def find_missing_route(self, path):
- # Return list of dicts to describe "routes" that should be
- # added when client requests the missing path given as an
- # argument. The dicts should look like:
- #
- # {
- # "method": "GET",
- # "path": "/foo/bar",
- # "callback": foo_bar_db,
- # }
- #
- # The "method" is the HTTP method, and defaults to "GET" if not given.
- # "path" is the path component of the URL being requested. It may use
- # Bottle.py patterns such as "/foo/<id>" for more powerful matching.
- # (Note that this means that if Bottle gets replaced, the patterns may
- # have to be reimplemented.)
- #
- # The list may be empty if no routes are to be added.
-
- return []
diff --git a/apifw/bottleapp.py b/apifw/bottleapp.py
deleted file mode 100644
index b248858..0000000
--- a/apifw/bottleapp.py
+++ /dev/null
@@ -1,229 +0,0 @@
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import logging
-import re
-import time
-
-
-import bottle
-import Crypto.PublicKey.RSA
-import jwt
-
-
-import apifw
-
-
-class BottleLoggingPlugin(apifw.HttpTransaction):
-
- # This a binding between Bottle and apifw.HttpTransaction, as a
- # Bottle plugin. We arrange the perform_transaction method to be
- # called with the right arguments, in particular the callback
- # function to call. We also provide Bottle specific methods to log
- # the HTTP request and response, and to amend the response by
- # adding a Date header.
-
- def apply(self, callback, route):
-
- def wrapper(*args, **kwargs):
- try:
- return self.perform_transaction(callback, *args, **kwargs)
- except bottle.HTTPError as e:
- self._log_error(e)
- raise e
- except BaseException as e:
- self._log_error(e)
- raise bottle.HTTPError(500, body=str(e))
-
- return wrapper
-
- def amend_response(self):
- rfc822 = time.strftime('%a, %d %b %Y %H:%M:%S %z')
- bottle.response.set_header('Date', rfc822)
-
- def construct_request_log(self):
- r = bottle.request
- return {
- 'method': r.method,
- 'path': r.path,
- }
-
- def construct_response_log(self):
- r = bottle.response
- return {
- 'status': r.status_code,
- 'text': r.body,
- }
-
-
-class BottleAuthorizationPlugin:
-
- route_pat = re.compile(r'<[^>]*>')
-
- def __init__(self):
- self.pubkey = None
- self.iss = None
- self.aud = None
-
- def set_token_signing_public_key(self, pubkey):
- self.pubkey = Crypto.PublicKey.RSA.importKey(pubkey)
-
- def set_expected_issuer(self, iss):
- self.iss = iss
-
- def set_expected_audience(self, aud):
- self.aud = aud
-
- def apply(self, callback, route):
-
- def wrapper(*args, **kwargs):
- if self.is_authorized(route):
- return callback(*args, **kwargs)
-
- self.raise_unauthorized('Something went wrong')
-
- return wrapper
-
- def is_authorized(self, route):
- value = self.get_authorization_header(bottle.request)
- token = self.parse_authorization_header(value)
- claims = self.parse_token(token)
- self.check_issuer(claims)
- return self.scope_allows_route(claims['scope'], route)
-
- def get_authorization_header(self, request):
- value = request.get_header('Authorization', '')
- if not value:
- self.raise_unauthorized('No Authorization header')
- logging.debug('Request has Authorization header: good')
- return value
-
- def parse_authorization_header(self, value):
- words = value.split()
- if len(words) != 2 or words[0].lower() != 'bearer':
- self.raise_unauthorized('Authorization should be "Bearer TOKEN"')
- logging.debug(
- 'Request Authorization header looks like a bearer token: good')
- return words[1]
-
- def parse_token(self, token):
- try:
- token = apifw.decode_token(token, self.pubkey, audience=self.aud)
- logging.debug('Request Authorization token can be decoded: good')
- return token
- except jwt.InvalidTokenError as e:
- self.raise_unauthorized(str(e))
-
- def check_issuer(self, claims):
- if claims['iss'] != self.iss:
- self.raise_unauthorized(
- 'Expected issuer %s, got %s' % (self.iss, claims['iss']))
- logging.debug('Token issuer is correct: good')
-
- def scope_allows_route(self, claim_scopes, route):
- scopes = claim_scopes.split(' ')
- route_scope = self.get_scope_for_route(route['method'], route['rule'])
- if route_scope in scopes:
- logging.debug(
- 'Route scope %s is in scopes %r', route_scope, scopes)
- return True
- logging.error(
- 'Route scope %s is NOT in scopes %r', route_scope, scopes)
- return False
-
- def get_scope_for_route(self, method, rule):
- scope = re.sub(self.route_pat, 'id', rule)
- scope = scope.replace('/', '_')
- scope = 'uapi%s_%s' % (scope, method)
- return scope.lower()
-
- def raise_unauthorized(self, explanation):
- headers = {
- 'WWW-Authenticate': 'Bearer',
- }
- raise bottle.HTTPError(401, body=explanation, headers=headers)
-
-
-class BottleApplication:
-
- # Provide the interface to bottle.Bottle that we need.
- # Specifically, we set up a hook to call the
- # Api.find_missing_route method when a request would otherwise
- # return 404, and we add the routes it returns to Bottle so that
- # they are there for this and future requests.
-
- def __init__(self, bottleapp, api):
- self._bottleapp = bottleapp
- self._bottleapp.add_hook('before_request', self._add_missing_route)
- self._api = api
-
- def add_plugin(self, plugin):
- self._bottleapp.install(plugin)
-
- def _add_missing_route(self):
- try:
- self._bottleapp.match(bottle.request.environ)
- except bottle.HTTPError:
- routes = self._api.find_missing_route(bottle.request.path)
- if routes:
- for route in routes:
- callback = self._callback_with_body(route['callback'])
- route_dict = {
- 'method': route.get('method', 'GET'),
- 'path': route['path'],
- 'callback': callback,
- }
- self._bottleapp.route(**route_dict)
- else:
- raise
-
- def _callback_with_body(self, callback):
- def wrapper(*args, **kwargs):
- content_type, body = self._get_request_body()
- response = callback(content_type, body, *args, **kwargs)
- return bottle.HTTPResponse(
- status=response['status'], body=response['body'],
- headers=response['headers'])
- return wrapper
-
- def _get_request_body(self):
- content_type = bottle.request.get_header('Content-Type')
- if content_type == 'application/json':
- body = bottle.request.json
- else:
- body = bottle.request.body.read()
- return content_type, body
-
-
-def create_bottle_application(api, logger, config):
- # Create a new bottle.Bottle application, set it up, and return it
- # so that gunicorn can execute it from the main program.
-
- bottleapp = bottle.Bottle()
- app = BottleApplication(bottleapp, api)
-
- plugin = BottleLoggingPlugin()
- if logger:
- plugin.set_dict_logger(logger)
- app.add_plugin(plugin)
-
- authz = BottleAuthorizationPlugin()
- authz.set_token_signing_public_key(config['token-public-key'])
- authz.set_expected_issuer(config['token-issuer'])
- authz.set_expected_audience(config['token-audience'])
- app.add_plugin(authz)
-
- return bottleapp
diff --git a/apifw/http.py b/apifw/http.py
deleted file mode 100644
index e859c07..0000000
--- a/apifw/http.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-HTTP_OK = 200
-HTTP_CREATED = 201
-HTTP_NOT_FOUND = 404
-HTTP_BAD_REQUEST = 400
-
-
-class HttpTransaction:
-
- # This class executes HTTP transactions. It is called by the web
- # framework, by some method that we don't specify. See the
- # BottleLoggingPlugin to use Bottle. This class is independent
- # from Bottle. The entry point is the "perform_tranaction" method,
- # which is tgiven the callback to call for the HTTP request that
- # has been made. This class makes the call, but also logs the
- # request and response, and catches and logs any excpetion.
- # Actually, the request and response get logged by a sub-class of
- # this class, which is supposed to define the
- # "construct_request_log" and "construct_response_log" methods.
- #
- # Log messages, as used by this class, are dicts, with no
- # particular specified keys, although "msg_type" is added by this
- # class. The request and response methods are supposed to return
- # dicts that get merged with what this class provides. The merged
- # result is then actually logged.
-
- def __init__(self):
- self._logger = lambda log, **kwargs: None
-
- def construct_request_log(self):
- # Override in subclass, as necessary.
- return {}
-
- def construct_response_log(self):
- # Override in subclass, as necessary.
- return {}
-
- def amend_response(self):
- # Make any necessary changes to the response.
- # Override in subclass, as necessary.
- pass
-
- def set_dict_logger(self, logger):
- # Set function to actually do logging.
- self._logger = logger
-
- def _log_request(self):
- log = {
- 'msg_type': 'http-request',
- }
- self._logger(self._combine_dicts(log, self.construct_request_log()))
-
- def _log_response(self):
- log = {
- 'msg_type': 'http-response',
- }
- self._logger(self._combine_dicts(log, self.construct_response_log()))
-
- def _log_error(self, exc):
- log = {
- 'msg_type': 'error',
- 'msg_text': 'caught exception at runtime',
- 'exception': str(exc),
- }
- self._logger(self._combine_dicts(log), stack_info=True)
-
- def _combine_dicts(self, *dicts):
- log = {}
- for d in dicts:
- log.update(d)
- return log
-
- def perform_transaction(self, callback, *args, **kwargs):
- try:
- self._log_request()
- data = callback(*args, **kwargs)
- self.amend_response()
- self._log_response()
- return data
- except SystemExit:
- # If we're exiting, we exit. No need to log an error.
- raise
- except BaseException as e:
- # Everything else results in an error logged.
- self._log_error(e)
- raise
-
-
-class Response:
-
- def __init__(self, values):
- self._dict = {}
- self._keys = ['status', 'headers', 'body']
- for key in self._keys:
- self[key] = ''
- for key, value in values.items():
- self[key] = value
-
- def __setitem__(self, key, value):
- assert key in self._keys
- self._dict[key] = value
-
- def __getitem__(self, key):
- return self._dict[key]
diff --git a/apifw/token.py b/apifw/token.py
deleted file mode 100644
index 9e3df8e..0000000
--- a/apifw/token.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import jwt
-
-
-JWT_SIGNING_ALGORITHM = 'RS512'
-
-
-def create_token(claims, signing_key):
- return jwt.encode(
- claims,
- signing_key.exportKey('PEM'),
- algorithm=JWT_SIGNING_ALGORITHM)
-
-
-def decode_token(token, key, audience):
- return jwt.decode(
- token,
- key=key.exportKey('OpenSSH'),
- audience=audience)
diff --git a/apifw/token_tests.py b/apifw/token_tests.py
deleted file mode 100644
index 12871c0..0000000
--- a/apifw/token_tests.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import time
-import unittest
-
-import Crypto.PublicKey.RSA
-import jwt
-
-import apifw
-
-
-# We generate keys here, once, so we only need to do it once.
-
-token_signing_key = Crypto.PublicKey.RSA.generate(1024)
-wrong_signing_key = Crypto.PublicKey.RSA.generate(1024)
-
-
-class TokenTests(unittest.TestCase):
-
- def make_claimes(self, now):
- return {
- 'iss': 'https://idp.example.com',
- 'sub': 'subject-uuid',
- 'aud': 'audience-uuid',
- 'exp': now + 3600,
- 'scope': 'openid person_resource_id uapi_orgs_get uapi_orgs_post'
- }
-
- def test_valid_token_is_valid(self):
- now = int(time.time())
- claims = self.make_claimes(now)
- encoded = apifw.create_token(claims, token_signing_key)
- self.assertTrue(isinstance(encoded, bytes))
- self.assertEqual(len(encoded.split(b'.')), 3)
-
- decoded = apifw.decode_token(
- encoded,
- token_signing_key,
- claims['aud'])
- self.assertEqual(decoded, claims)
-
- def test_expired_token_is_invalid(self):
- now = int(time.time())
- claims = self.make_claimes(now - 86400)
- encoded = apifw.create_token(claims, token_signing_key)
- with self.assertRaises(jwt.ExpiredSignatureError):
- apifw.decode_token(
- encoded,
- token_signing_key,
- claims['aud'])
-
- def test_wronly_signed_token_is_invalid(self):
- now = int(time.time())
- claims = self.make_claimes(now - 86400)
- encoded = apifw.create_token(claims, wrong_signing_key)
- with self.assertRaises(jwt.DecodeError):
- apifw.decode_token(
- encoded,
- token_signing_key,
- claims['aud'])
diff --git a/apitest.key b/apitest.key
deleted file mode 100644
index e3518cd..0000000
--- a/apitest.key
+++ /dev/null
@@ -1,51 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIJKgIBAAKCAgEA2h4IxT6ukFAzI1aqd7/zERhgvcxFwNd09BMJb6GROSznk1a4
-upjDcGqsUMjBIted+bAFbknW+lYGLs5TEkZxDhL2nU0m5iI9kxLx969L33d99tdC
-OFg00hM3T2zIkEYZjBvhG1vFCiBFgKycsruk0CRjJgLDTKa+pl4uQEtEpmUnvgyu
-iWOBIAustoPchrDzaF2LSMHXHTVENMPxRnZXhRj9/xcx4+yht6i9knq6cNBA2BIR
-kdWVUCKbTQDsF3ZAsHT3ZmXkRLN/1txZdrAH/DNMB4FHuqk8zgv3Gm5qBjZUzbkI
-fWCr92OA94Gmb4VLySHFTrsLT+BMAXBY4l9MKt56GLZ51WRIm91CBVeekQjJiaXw
-k8rU2ArjTejB6CdStXzdfEmCo9YUdHINHqjXJx6YcftZQOHNpiDTCjaCGh+4UiKz
-Yr4loWjzuD9oKDA4PEXgrFFIvtAAGeVWO9S+kes59UPEHl9f9heFlPgHmHeuBwY+
-GfNPDYAUtPcyQDVSfJPrqf5LsfHcLBwuyLTRjUAJGKlcPWWBlGDalJMwVK8R7WUX
-B/JORdn5OUuckFiqD+TeqZC1uF2H10xXBSf6WYNFUES8YKMMfdD+xFub8Iu1kJ30
-jXUVtVjFRc4MKRaUv+6eMDLnA/T9+aKURt+k2OTdkwcvhmH1CEAVV2ew1akCAwEA
-AQKCAgEAxisZftOnWBh1jbrU4D22Qibq5iYsjbtzV7ngDds2DUNeFsBoz6exkXZp
-nm/3AYfy0IL7PCu8NO9paKcuVGFJoCbchygsmlQrq29ABe/vOFXhTR5f3L9PJjot
-O20zf9kgpupBiFDFYaDWZMTvDXhskmss5cEG3aJ1fsP8s49vDNrE0+fDv7F3BL12
-qtB80Kb+TykGPhFXNwNJN8N6d7FXbOa7BkN1oYZBm5KkwevdblfXQjiQW/Y4VXlL
-rheTaPGYbnmmuRMD5ONM19KVOb9PUfTtM7hiihXu16mJVStSCtjcDZj6PKdTNk3i
-Q3040QPDSjbzg8duzKCVjY0cRHeew3qKLAKnyZDFqChU9mTvbZSx/+SdOcLRmBIE
-Lcgopd5ZLOMVDgYukt2Yn98s8WlNL4vXxtyUFJlcT5PEwI2Ctq8I+8cY/1EKgmb4
-tyyZ2f6arfsKrlBstauwvQXy9nz6nd/lpQvph0ZvaY3QPRekkdKMTNtdk/tqXeEG
-QOYvvbFA3QKJwg+pWU+3NK8maEYaxm2eRTTXQsLeJHkDlRkUAg1s7UumIiIvlv/Z
-WOiQq8H34cEGnMUrFhFkoOHAFabsHH68P0VrOiMoRUtrBhekQUrvBdnzAE7gLToo
-JRtnQmwTxqZM4KRozxIfAJYM+3qWdfBi/pAk1bhrsgHQRE1ZRI0CggEBAOTh08Wn
-hPQ2d0w4hL0GZPP0RnCc4u/VLfoHwMw3MsXd8zbKljChfDwKDSsgoW0JnptvZENU
-SyPClVRCVCpOLRuZvrA/TlavQ4kpV/lrCcoIp3P+/tH/bPJDJlobAFE1LpbxJT+1
-PaMCo1hbblTc6Hir/a5DEJ4Wwaz6XMjMQu0U9LWZquHXLUoHScVDVdCJyp54EAF5
-PCddI9subfRcbOZ4aTVWADK06T0Y0PXse87c0HG8k0J+JO/l32Y3fzgOndAaIehr
-KgWiLWx4FuZ/fR95omCyKgAMvaXBy/W2wobeZzdUagp3+I5haw7pwi3QsW2cKr1B
-ORtXR8WMoWtEVBcCggEBAPP1s6jy+LtL+GLFRrDzv1t2GeAaehf4sO8GQzCyFRpU
-qI/nUuSQ44P2BsgoB30iWk1X3NHvvdboOTNJE8PhmY6URfIMUNzCfX6KpS3B08fn
-gxToFRLtnOXXOe5Q05jkM2stM7Q3LOvr9NIeC0+Q69bTOh844LL7/K8CPgWPT8Md
-fqLIc48Y5dXAW2WHW0tZlnCnRWkMfdojbX6khwt6UEvSeed3VKV5BDghmKf9Lz08
-K2xoRHButM1zYyee3Itn5gLVaGMOe+peZf6ItYDLopP0fPnEjEAjIz4He1xCcOMa
-VbtaaKLHvmGlOc8xfeqTswrsZ2TX7r5YMbFjmiYjfD8CggEAJjFo3Tqu/PF5xsZH
-oCNJBUxl1LTZSZwRJ//TNEChwFLhGuuDVGoeCQbEW8X+KevJA7b6zCFsyHLX6E+J
-K+YPsONe5popwF1Or7yuaXhrEcOP7dNHQlOVIngCFlcbHnH5bEahKJhdyK3QBBZ9
-uruCL2DD3ChkxXyWpP7CLN+o61br3sHdugHmFMxSixBJaZsUrIzsXtKULx1jtldx
-Ea26nlrJc5T+Q3fc080oUWE857ABOHl3OUlDcKSzOqNYH0qRGwDBV79KK9Z8LfV8
-HMp5Xp81cV1JlOiLXPWRy1bL7yV9o8X6S/TpDRlEfCCVvn9snBXLK/mORfmyiEyH
-QxcL0QKCAQEA2dyLys7grXKUqLAABzpFo0n+pZE/g3TFnU98ZVpDWjZMKeassg1q
-AIiPWePVfDxXZEaYnqp4YBkWK+SQ0BcB6MAlDplNBThylbT++bPkitsGxn7TgwnL
-Wb4wr9BihmEUQhwl1kSHy0/2XEYUV8PVuQz9FLDYiT5bU3avKIvo8Re/5WMZP6s+
-ZPrZI/wS3WFt5cCbTcqoAUwuFjCboPZkCrI1xy1b3EIMMIxgJXUG4KqBJNigdb+H
-mwn2fIVz8tKgJ9uo0v02UABpGTvAyvoPgA2QJgUOMqCuclCAK83xvf3gneWJGAVE
-0TKaQ5uxFPE9rP2dAAON41IjXoTSPkjmRQKCAQEAta44u5SJyJ1A0WGQa8BJ/sSZ
-HCBcuV+GsKW6l+2NHAqQqJ3ouPNC/jnoGgKtaAfSpIJLKahO6WDupQ/yubnU6LA4
-jDtfW5SsfkgJEvvCKi98kftNNuVxS+cPUncaeaEXjuUNZPI8Vpb1kHKN4V4FVN0m
-P0tgHeNgfNMpCqfLxcWcO0Wukn+azmxP95jARMwwAn6VK+/fHlYGmODBH7w1NZFp
-p0vFqoUJqdm++rF9tUdwfwK8gtfZ2ZgnDpOe8uoWwX5EfMsxMXB6Twf6jkLDUHtj
-QXXJ+wsfMv/l/8f4ScuAfb/pwmWtvI3YiTGcpkvJHweacA6fRzgmm75TKWNixQ==
------END RSA PRIVATE KEY----- \ No newline at end of file
diff --git a/apitest.key.pub b/apitest.key.pub
deleted file mode 100644
index 397dd2c..0000000
--- a/apitest.key.pub
+++ /dev/null
@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDaHgjFPq6QUDMjVqp3v/MRGGC9zEXA13T0EwlvoZE5LOeTVri6mMNwaqxQyMEi1535sAVuSdb6VgYuzlMSRnEOEvadTSbmIj2TEvH3r0vfd33210I4WDTSEzdPbMiQRhmMG+EbW8UKIEWArJyyu6TQJGMmAsNMpr6mXi5AS0SmZSe+DK6JY4EgC6y2g9yGsPNoXYtIwdcdNUQ0w/FGdleFGP3/FzHj7KG3qL2Serpw0EDYEhGR1ZVQIptNAOwXdkCwdPdmZeREs3/W3Fl2sAf8M0wHgUe6qTzOC/cabmoGNlTNuQh9YKv3Y4D3gaZvhUvJIcVOuwtP4EwBcFjiX0wq3noYtnnVZEib3UIFV56RCMmJpfCTytTYCuNN6MHoJ1K1fN18SYKj1hR0cg0eqNcnHphx+1lA4c2mINMKNoIaH7hSIrNiviWhaPO4P2goMDg8ReCsUUi+0AAZ5VY71L6R6zn1Q8QeX1/2F4WU+AeYd64HBj4Z808NgBS09zJANVJ8k+up/kux8dwsHC7ItNGNQAkYqVw9ZYGUYNqUkzBUrxHtZRcH8k5F2fk5S5yQWKoP5N6pkLW4XYfXTFcFJ/pZg0VQRLxgowx90P7EW5vwi7WQnfSNdRW1WMVFzgwpFpS/7p4wMucD9P35opRG36TY5N2TBy+GYfUIQBVXZ7DVqQ== \ No newline at end of file
diff --git a/apitest.py b/apitest.py
deleted file mode 100644
index b3987d8..0000000
--- a/apitest.py
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/usr/bin/python3
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# This is a test application for apifw, used by the apifw.yarn test
-# suite. It also acts as an example of how apifw can be used.
-
-import logging
-import os
-
-import yaml
-
-import apifw
-
-
-# apifw requires a class to define routes on demand. This interface is provided
-# by the apifw.Api class. This interface consists of a single method
-# find_missing_route. In addition callback methods for particular routes
-# may also be present.
-
-class Api(apifw.Api):
-
- def find_missing_route(self, path):
- logging.info('find_missing_route called!\n')
- return [
- {
- 'path': '/version',
- 'callback': self.version,
- },
- {
- 'method': 'PUT',
- 'path': '/upload',
- 'callback': self.upload,
- },
- ]
-
- def version(self, content_type, body):
- return apifw.Response({
- 'status': apifw.HTTP_OK,
- 'body': 'version: 4.2',
- 'headers': {
- 'Content-Type': 'text/plain',
- },
- })
-
- def upload(self, content_type, body):
- return apifw.Response({
- 'status': apifw.HTTP_OK,
- 'body': 'thank you for %s\n' % body.decode('ascii'),
- 'headers': {
- 'Content-Type': 'text/plain',
- },
- })
-
-
-# We want logging. gunicorn provides logging, but only of its own
-# stuff, and if we log something ourselves, using logging.debug and
-# its sibling functions, they don't end up in the log file defined by
-# gunicorn.
-#
-# Instead a separate logger is defined. The logging in
-# apifw.HttpTransaction are based on logging *dicts*, with arbitrary
-# key/value pairs. This is because Lars liked log files that are easy
-# to process programmatically. To use this, we define a "dict logger"
-# function, which we tell apifw to use; by default, there is no
-# logging. We also configure the Python logging library to write a log
-# file; the name of the log file will be gotten from the APITEST_LOG
-# environment variable.
-#
-# Ideally, we would parse command line arguments and a configuration
-# file here, instead of using environment variables. Unfortunately,
-# gunicorn wants to hog all of the command line for itself.
-#
-# It has been concluded that the sensible way is for a web app to no
-# have command line options and to read configuration file from a
-# fixed location, or have the name of the configuration file given
-# using an environment variable.
-#
-# See also <https://stackoverflow.com/questions/8495367/>.
-
-def dict_logger(log, stack_info=None):
- logging.info('Start log entry')
- for key in sorted(log.keys()):
- logging.info(' %r=%r', key, log[key])
- logging.info('Endlog entry')
- if stack_info:
- logging.info('Traceback', exc_info=True)
-
-
-logfile = os.environ.get('APITEST_LOG')
-if logfile:
- logging.basicConfig(filename=logfile, level=logging.DEBUG)
-
-
-
-# Here is some configuaration for the framework.
-#
-# To validate the access tokens we'll be using in the API, we need to
-# tell apifw which public key to use. We get that from an environment
-# variable. It should be in OpenSSH public key format.
-#
-# The framework also needs to know the "audience" for which a token is
-# created. We get that from the environment as well.
-
-config = {
- 'token-public-key': os.environ['APITEST_PUBKEY'],
- 'token-audience': os.environ['APITEST_AUD'],
- 'token-issuer': os.environ['APITEST_ISS'],
-}
-
-
-# Here is the magic part. We create an instance of our Api class, and
-# create a new Bottle application, using the
-# apifw.create_bottle_application function. We assign the Bottle app
-# to the variable "app", and when we start this program using
-# gunicorn, we tell it to use the "app" variable.
-#
-# gunicorn3 --bind 127.0.0.1:12765 apitest:app
-#
-# gunicorn and Bottle then collaborate, using mysterious, unclear, and
-# undocumented magic to run a web service. We don't know, and
-# hopefully don't need to care, how the magic works.
-
-api = Api()
-app = apifw.create_bottle_application(api, dict_logger, config)
-
-# If we are running this program directly with Python, and not via
-# gunicorn, we can use the Bottle built-in debug server, which can
-# make some things easier to debug.
-
-if __name__ == '__main__':
- print('running in debug mode')
- app.run(host='127.0.0.1', port=12765)
diff --git a/check b/check
index 13d1b1e..592539f 100755
--- a/check
+++ b/check
@@ -19,9 +19,8 @@ set -eu
python3 -m CoverageTestRunner --ignore-missing-from=without-tests
-pep8 apifw slog qvarn
-pylint3 -j0 --rcfile pylint.conf apifw slog qvarn
+pep8 qvarn
+pylint3 -j0 --rcfile pylint.conf qvarn
-yarn apifw.yarn "$@"
yarn yarns/*.yarn \
-s yarns/lib.py --shell python2 --shell-arg '' --cd-datadir "$@"
diff --git a/run-apitest b/run-apitest
deleted file mode 100755
index 4455e33..0000000
--- a/run-apitest
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/sh
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-set -eu
-
-
-log="$1"
-token="$2"
-
-
-tmp="$(mktemp -d)"
-
-cleanup()
-{
- rm -rf "$tmp"
-}
-
-trap cleanup EXIT
-
-export APITEST_ISS=testissuer
-export APITEST_AUD=aud
-export APITEST_LOG="$log"
-
-scopes="
-uapi_version_get
-uapi_upload_put
-"
-
-./generate-rsa-key "$tmp/key"
-./create-token "$tmp/key" "$APITEST_ISS" "$APITEST_AUD" "$scopes" > "$token"
-export APITEST_PUBKEY="$(cat "$tmp/key.pub")"
-
-gunicorn --bind 127.0.0.1:12765 -p "$tmp/pid" -w1 \
- --log-file "$log" --log-level=debug apitest:app
diff --git a/slog-errors b/slog-errors
deleted file mode 100755
index 920e87d..0000000
--- a/slog-errors
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-#
-# Read Qvarn structured log files, look for errors, and output all
-# messages related to a failed HTTP request (status code 5xx).
-
-
-import json
-import sys
-
-
-def parse_log_file(filename):
- with open(filename) as f:
- for line in f:
- yield json.loads(line)
-
-
-def find_contexts(msgs):
- return set(get_context(msg) for msg in msgs)
-
-
-def get_context(msg):
- return msg['_context'], msg['_process_id'], msg['_thread_id']
-
-
-def is_in_context(msg, context):
- return get_context(msg) == context
-
-
-def filter_msgs(msgs, func):
- for msg in msgs:
- if func(msg):
- yield msg
-
-def has_error_status(msgs):
- for msg in msgs:
- if msg['msg_type'] == 'error':
- return True
- if msg['msg_type'] == 'http-response' and msg['status'] >= 400:
- return True
- if '_traceback' in msg:
- return True
- return False
-
-
-def show_msgs(msgs):
- sys.stdout.write(json.dumps(msgs))
- sys.stdout.write('\n')
-
-
-msgs = []
-for filename in sys.argv[1:]:
- for msg in parse_log_file(filename):
- msg['_filename'] = filename
- msgs.append(msg)
-
-
-contexts = find_contexts(msgs)
-for context in contexts:
- context_msgs = list(filter_msgs(msgs, lambda m: is_in_context(m, context)))
- if has_error_status(context_msgs):
- show_msgs(context_msgs)
diff --git a/slog-pretty b/slog-pretty
deleted file mode 100755
index 80f07fb..0000000
--- a/slog-pretty
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-
-import json
-import sys
-
-import yaml
-
-
-def text_representer(dumper, data):
- if '\n' in data:
- return dumper.represent_scalar(
- u'tag:yaml.org,2002:str', data, style='|')
- return dumper.represent_scalar(
- u'tag:yaml.org,2002:str', data, style='')
-
-yaml.add_representer(str, text_representer)
-yaml.add_representer(unicode, text_representer)
-
-
-def dump(f):
- for line in f:
- obj = json.loads(line.strip())
- yaml.dump(
- obj, stream=sys.stdout, indent=4, default_flow_style=False,
- explicit_start=True, explicit_end=True)
-
-
-if len(sys.argv) == 1:
- dump(sys.stdin)
-else:
- for filename in sys.argv[1:]:
- with open(filename) as f:
- dump(f)
diff --git a/slog/__init__.py b/slog/__init__.py
deleted file mode 100644
index 45f93b0..0000000
--- a/slog/__init__.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright (C) 2017 Lars Wirzenius
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-from .counter import Counter
-
-from .slog import (
- StructuredLog,
- NullSlogWriter,
- FileSlogWriter,
- SyslogSlogWriter,
- SlogHandler,
- hijack_logging,
-)
-
-from .slog_filter import (
- FilterAllow,
- FilterDeny,
- FilterFieldHasValue,
- FilterFieldValueRegexp,
- FilterHasField,
- FilterInclude,
- FilterAny,
- construct_log_filter,
-)
diff --git a/slog/counter.py b/slog/counter.py
deleted file mode 100644
index 4760fe8..0000000
--- a/slog/counter.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2015, 2016 Suomen Tilaajavastuu Oy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import _thread
-
-
-class Counter:
-
- '''A threadsafe incrementing counter.'''
-
- def __init__(self):
- self._lock = _thread.allocate_lock()
- self._counter = 0
-
- def get(self):
- return self._counter
-
- def increment(self):
- with self._lock:
- self._counter += 1
- return self._counter
diff --git a/slog/counter_tests.py b/slog/counter_tests.py
deleted file mode 100644
index 2c19fe1..0000000
--- a/slog/counter_tests.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright 2016 QvarnLabs Ltd
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import threading
-import unittest
-
-import slog
-
-
-class CounterTests(unittest.TestCase):
-
- def test_counts_to_a_million_in_ten_threads(self):
- max_count = 100000
- num_threads = 2
- counter = slog.Counter()
-
- threads = []
- for _ in range(num_threads):
- ct = CountThread(max_count, counter)
- ct.start()
- threads.append(ct)
- for ct in threads:
- ct.join()
-
- self.assertEqual(counter.get(), max_count * num_threads)
-
-
-class CountThread(threading.Thread):
-
- def __init__(self, n, counter):
- super(CountThread, self).__init__()
- self.n = n
- self.counter = counter
-
- def run(self):
- for _ in range(self.n):
- self.counter.increment()
diff --git a/slog/slog.py b/slog/slog.py
deleted file mode 100644
index 81d2c61..0000000
--- a/slog/slog.py
+++ /dev/null
@@ -1,278 +0,0 @@
-# slog.py - structured logging
-#
-# Copyright 2016-2017 QvarnLabs Ab
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import datetime
-import json
-import logging
-import os
-import time
-import traceback
-import _thread
-import syslog
-
-import slog
-
-
-class StructuredLog:
-
- '''A structured logging system.
-
- A structured log is one that can be easily parsed
- programmatically. Traditional logs are free form text, usually
- with a weakly enforced line structure and some minimal metadata
- prepended to each file. This class produces JSON records instead.
-
- See the separate manual for more background and examples of how to
- use this system.
-
- '''
-
- def __init__(self):
- self._msg_counter = slog.Counter()
- self._context = {}
- self._writers = []
-
- def close(self):
- for writer, _ in self._writers:
- writer.close()
- self._writers = []
-
- def add_log_writer(self, writer, filter_rule):
- self._writers.append((writer, filter_rule))
-
- def set_context(self, new_context):
- thread_id = self._get_thread_id()
- self._context[thread_id] = new_context
-
- def reset_context(self):
- thread_id = self._get_thread_id()
- self._context[thread_id] = None
-
- def log(self, msg_type, **kwargs):
- exc_info = kwargs.pop('exc_info', False)
-
- log_obj = {
- 'msg_type': msg_type,
- }
- for key, value in kwargs.items():
- log_obj[key] = self._convert_value(value)
-
- self._add_extra_fields(log_obj, exc_info)
- for writer, filter_rule in self._writers:
- if filter_rule.allow(log_obj):
- writer.write(log_obj)
-
- def _convert_value(self, value):
- # Convert a value into an form that's safe to write. Meaning,
- # it can't be binary data, and it is UTF-8 compatible, if it's
- # a string of any sort.
- #
- # Note that we do not need to be able to parse the value back
- # again, we just need to write it to a log file in a form that
- # the user will understand. At least for now.
- #
- # We can't do this while encoding JSON, because the Python
- # JSON library doesn't seem to allow us to override how
- # encoding happens for types it already knows about, only for
- # other types of values.
-
- converters = {
- int: self._nop_conversion,
- float: self._nop_conversion,
- bool: self._nop_conversion,
- str: self._nop_conversion,
- type(None): self._nop_conversion,
-
- list: self._convert_list_value,
- dict: self._convert_dict_value,
- tuple: self._convert_tuple_value,
- }
-
- value_type = type(value)
- assert value_type in converters, \
- 'Unknown data type {}'.format(value_type)
- func = converters[type(value)]
- converted = func(value)
- return converted
-
- def _nop_conversion(self, value):
- return value
-
- def _convert_list_value(self, value):
- return [self._convert_value(item) for item in value]
-
- def _convert_tuple_value(self, value):
- return tuple(self._convert_value(item) for item in value)
-
- def _convert_dict_value(self, value):
- return {
- self._convert_value(key): self._convert_value(value[key])
- for key in value
- }
-
- def _add_extra_fields(self, log_obj, stack_info):
- log_obj['_msg_number'] = self._get_next_message_number()
- log_obj['_timestamp'] = self._get_current_timestamp()
- log_obj['_process_id'] = self._get_process_id()
- log_obj['_thread_id'] = self._get_thread_id()
- log_obj['_context'] = self._context.get(self._get_thread_id())
- if stack_info:
- log_obj['_traceback'] = self._get_traceback()
-
- def _get_next_message_number(self):
- return self._msg_counter.increment()
-
- def _get_current_timestamp(self):
- return datetime.datetime.utcnow().isoformat(' ')
-
- def _get_process_id(self):
- return os.getpid()
-
- def _get_thread_id(self):
- return _thread.get_ident()
-
- def _get_traceback(self):
- return traceback.format_exc()
-
-
-class SlogWriter: # pragma: no cover
-
- def write(self, log_obj):
- raise NotImplementedError()
-
- def close(self):
- raise NotImplementedError()
-
-
-class NullSlogWriter(SlogWriter): # pragma: no cover
-
- def write(self, log_obj):
- pass
-
- def close(self):
- pass
-
-
-class FileSlogWriter(SlogWriter):
-
- def __init__(self):
- self._log_filename = None
- self._log_file = None
- self._bytes_max = None
- self._encoder = json.JSONEncoder(sort_keys=True)
-
- def set_max_file_size(self, bytes_max):
- self._bytes_max = bytes_max
-
- def get_filename(self):
- return self._log_filename
-
- def get_rotated_filename(self, now=None):
- prefix, suffix = os.path.splitext(self._log_filename)
- if now is None: # pragma: no cover
- now = time.localtime()
- else:
- now = tuple(list(now) + [0]*9)[:9]
- timestamp = time.strftime('%Y%m%dT%H%M%S', now)
- return '{}-{}{}'.format(prefix, timestamp, suffix)
-
- def set_filename(self, filename):
- self._log_filename = filename
- self._log_file = open(filename, 'a')
-
- def write(self, log_obj):
- if self._log_file:
- self._write_message(log_obj)
- if self._bytes_max is not None:
- self._rotate()
-
- def _write_message(self, log_obj):
- msg = self._encoder.encode(log_obj)
- self._log_file.write(msg + '\n')
- self._log_file.flush()
-
- def _rotate(self):
- pos = self._log_file.tell()
- if pos >= self._bytes_max:
- self._log_file.close()
- rotated = self.get_rotated_filename()
- os.rename(self._log_filename, rotated)
- self.set_filename(self._log_filename)
-
- def close(self):
- self._log_file.close()
- self._log_file = None
-
-
-class SyslogSlogWriter(SlogWriter): # pragma: no cover
-
- def write(self, log_obj):
- encoder = json.JSONEncoder(sort_keys=True)
- s = encoder.encode(log_obj)
- syslog.syslog(s)
-
- def close(self):
- pass
-
-
-class SlogHandler(logging.Handler): # pragma: no cover
-
- '''A handler for the logging library to capture into a slog.
-
- In order to capture all logging.* log messages into a structured
- log, configure the logging library to use this handler.
-
- '''
-
- def __init__(self, log):
- super(SlogHandler, self).__init__()
- self.log = log
-
- def emit(self, record):
- attr_names = {
- 'msg': 'msg_text',
- }
-
- log_args = dict()
- for attr in dir(record):
- if not attr.startswith('_'):
- value = getattr(record, attr)
- if not isinstance(value, (str, int, bool, float)):
- value = repr(value)
- log_args[attr_names.get(attr, attr)] = value
- self.log.log('logging.' + record.levelname, **log_args)
-
-
-def hijack_logging(log, logger_names=None): # pragma: no cover
- '''Hijack log messages that come via logging.* into a slog.'''
-
- handler = SlogHandler(log)
-
- for name in logger_names or []:
- logger = logging.getLogger(name)
- hijack_logger_handlers(logger, handler)
-
- logger = logging.getLogger()
- hijack_logger_handlers(logger, handler)
-
-
-def hijack_logger_handlers(logger, handler): # pragma: no cover
- logger.setLevel(logging.DEBUG)
- for h in logger.handlers:
- logger.removeHandler(h)
- logger.addHandler(handler)
diff --git a/slog/slog_filter.py b/slog/slog_filter.py
deleted file mode 100644
index 1115f05..0000000
--- a/slog/slog_filter.py
+++ /dev/null
@@ -1,117 +0,0 @@
-# slog_filter.py - structured logging filtering
-#
-# Copyright 2017 QvarnLabs Ab
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import re
-
-
-class FilterRule: # pragma: no cover
-
- def allow(self, log_obj):
- raise NotImplementedError()
-
-
-class FilterAllow(FilterRule):
-
- def allow(self, log_obj):
- return True
-
-
-class FilterDeny(FilterRule):
-
- def allow(self, log_obj):
- return False
-
-
-class FilterHasField(FilterRule):
-
- def __init__(self, field_name):
- self._field_name = field_name
-
- def allow(self, log_obj):
- return self._field_name in log_obj
-
-
-class FilterFieldHasValue(FilterRule):
-
- def __init__(self, field, value):
- self._field = field
- self._value = value
-
- def allow(self, log_obj):
- return self._field in log_obj and log_obj[self._field] == self._value
-
-
-class FilterFieldValueRegexp(FilterRule):
-
- def __init__(self, field, pattern):
- self._field = field
- self._pattern = pattern
-
- def allow(self, log_obj):
- if self._field not in log_obj:
- return False
- value = str(log_obj[self._field])
- return re.search(self._pattern, value) is not None
-
-
-class FilterInclude(FilterRule):
-
- def __init__(self, rule_dict, rule):
- self._rule = rule
- self._include = rule_dict.get('include', True)
-
- def allow(self, log_obj):
- allow = self._rule.allow(log_obj)
- return (self._include and allow) or (not self._include and not allow)
-
-
-class FilterAny(FilterRule):
-
- def __init__(self, rules):
- self._rules = rules
-
- def allow(self, log_obj):
- return any(rule.allow(log_obj) for rule in self._rules)
-
-
-def construct_log_filter(filters):
- if not filters:
- raise NoFilter()
- rules = []
- for spec in filters:
- rule = None
- if 'field' in spec:
- if 'value' in spec:
- rule = FilterFieldHasValue(spec['field'], spec['value'])
- elif 'regexp' in spec:
- rule = FilterFieldValueRegexp(
- spec['field'], spec['regexp'])
- elif 'field' in spec:
- rule = FilterHasField(spec['field'])
- else: # pragma: no cover
- rule = FilterAllow()
- if 'include' in spec:
- rule = FilterInclude(spec, rule)
- rules.append(rule)
- return FilterAny(rules)
-
-
-class NoFilter(Exception):
-
- def __init__(self):
- super(NoFilter, self).__init__('No log filter specified')
diff --git a/slog/slog_filter_tests.py b/slog/slog_filter_tests.py
deleted file mode 100644
index 401d6a0..0000000
--- a/slog/slog_filter_tests.py
+++ /dev/null
@@ -1,267 +0,0 @@
-# slog_filter_tests.py - unit tests for structured logging tests
-#
-# Copyright 2017 QvarnLabs Ab
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import unittest
-
-import slog
-
-
-class FilterHasFieldTests(unittest.TestCase):
-
- def test_allows_if_field_is_there(self):
- log_obj = {
- 'foo': False,
- }
- rule = slog.FilterHasField('foo')
- self.assertTrue(rule.allow(log_obj))
-
- def test_doesnt_allow_if_field_is_not_there(self):
- log_obj = {
- 'foo': False,
- }
- rule = slog.FilterHasField('bar')
- self.assertFalse(rule.allow(log_obj))
-
-
-class FilterFieldHasValueTests(unittest.TestCase):
-
- def test_allows_when_field_has_wanted_value(self):
- log_obj = {
- 'foo': 'bar',
- }
- rule = slog.FilterFieldHasValue('foo', 'bar')
- self.assertTrue(rule.allow(log_obj))
-
- def test_doesnt_allow_when_field_has_unwanted_value(self):
- log_obj = {
- 'foo': 'bar',
- }
- rule = slog.FilterFieldHasValue('foo', 'yo')
- self.assertFalse(rule.allow(log_obj))
-
-
-class FilterFieldValueRegexpTests(unittest.TestCase):
-
- def test_allows_when_value_matches_regexp(self):
- log_obj = {
- 'foo': 'bar',
- }
- rule = slog.FilterFieldValueRegexp('foo', 'b.r')
- self.assertTrue(rule.allow(log_obj))
-
- def test_allows_when_value_matches_regexp_if_stringified(self):
- log_obj = {
- 'foo': 400,
- }
- rule = slog.FilterFieldValueRegexp('foo', '4.*')
- self.assertTrue(rule.allow(log_obj))
-
- def test_doesnt_allow_when_field_isnt_there(self):
- log_obj = {
- 'blarf': 'yo',
- }
- rule = slog.FilterFieldValueRegexp('foo', 'b.r')
- self.assertFalse(rule.allow(log_obj))
-
- def test_doesnt_allow_when_value_doesnt_match_regexp(self):
- log_obj = {
- 'foo': 'yo',
- }
- rule = slog.FilterFieldValueRegexp('foo', 'b.r')
- self.assertFalse(rule.allow(log_obj))
-
-
-class FilterAllowTests(unittest.TestCase):
-
- def test_allows_always(self):
- rule = slog.FilterAllow()
- self.assertTrue(rule.allow(None))
-
-
-class FilterDenyTests(unittest.TestCase):
-
- def test_allows_denies(self):
- rule = slog.FilterDeny()
- self.assertFalse(rule.allow(None))
-
-
-class FilterIncludeTests(unittest.TestCase):
-
- def test_allows_if_rule_allows_and_no_include(self):
- allow = slog.FilterAllow()
- include = slog.FilterInclude({}, allow)
- self.assertTrue(include.allow(None))
-
- def test_denies_if_rule_denies_and_no_include(self):
- deny = slog.FilterDeny()
- include = slog.FilterInclude({}, deny)
- self.assertFalse(include.allow(None))
-
- def test_allows_if_rule_allows_and_include_is_true(self):
- allow = slog.FilterAllow()
- include = slog.FilterInclude({'include': True}, allow)
- self.assertTrue(include.allow({}))
-
- def test_denies_if_rule_denies_and_include_is_true(self):
- deny = slog.FilterDeny()
- include = slog.FilterInclude({'include': True}, deny)
- self.assertFalse(include.allow({}))
-
- def test_denies_if_rule_allows_and_include_is_false(self):
- allow = slog.FilterAllow()
- include = slog.FilterInclude({'include': False}, allow)
- self.assertFalse(include.allow({}))
-
- def test_allows_if_rule_denies_and_include_is_false(self):
- deny = slog.FilterDeny()
- include = slog.FilterInclude({'include': False}, deny)
- self.assertTrue(include.allow({}))
-
-
-class FilterAnyTests(unittest.TestCase):
-
- def test_allows_if_any_rule_allows(self):
- rules = [slog.FilterAllow()]
- any_rule = slog.FilterAny(rules)
- self.assertTrue(any_rule.allow(None))
-
- def test_denies_if_all_rules_deny(self):
- rules = [slog.FilterDeny(), slog.FilterAllow()]
- any_rule = slog.FilterAny(rules)
- self.assertTrue(any_rule.allow(None))
-
-
-class ConstructFilterTests(unittest.TestCase):
-
- def test_raises_error_if_no_filters(self):
- filters = []
- with self.assertRaises(Exception):
- slog.construct_log_filter(filters)
-
- def test_handles_field_wanted(self):
- filters = [
- {
- 'field': 'msg_type',
- },
- ]
-
- rule = slog.construct_log_filter(filters)
-
- allowed = {
- 'msg_type': 'info',
- }
- self.assertTrue(rule.allow(allowed))
-
- denied = {
- 'blah_blah': 'http-response',
- }
- self.assertFalse(rule.allow(denied))
-
- def test_handles_field_value_wanted(self):
- filters = [
- {
- 'field': 'msg_type',
- 'value': 'info',
- },
- ]
-
- rule = slog.construct_log_filter(filters)
-
- allowed = {
- 'msg_type': 'info',
- }
- self.assertTrue(rule.allow(allowed))
-
- denied = {
- 'msg_type': 'http-response',
- }
- self.assertFalse(rule.allow(denied))
-
- def test_handles_regexp_match_wanted(self):
- filters = [
- {
- 'field': 'status',
- 'regexp': '^4'
- },
- ]
-
- rule = slog.construct_log_filter(filters)
-
- allowed = {
- 'status': 400,
- }
- self.assertTrue(rule.allow(allowed))
-
- denied = {
- 'status': 200,
- }
- self.assertFalse(rule.allow(denied))
-
- def test_handles_not_included(self):
- filters = [
- {
- 'field': 'status',
- 'value': '400',
- 'include': False,
- },
- ]
-
- rule = slog.construct_log_filter(filters)
-
- allowed = {
- 'status': '200',
- }
- self.assertTrue(rule.allow(allowed))
-
- denied = {
- 'status': '400',
- }
- self.assertFalse(rule.allow(denied))
-
- def test_returns_compound_filter(self):
- filters = [
- {
- 'field': 'msg_type',
- },
- {
- 'field': 'msg_type',
- 'value': 'debug',
- },
- {
- 'field': 'status',
- 'regexp': '^4',
- 'include': False,
- },
- ]
-
- rule = slog.construct_log_filter(filters)
-
- allowed = {
- 'msg_type': 'info',
- }
- self.assertTrue(rule.allow(allowed))
-
- also_allowed = {
- 'msg_type': 'debug',
- }
- self.assertTrue(rule.allow(also_allowed))
-
- denied = {
- 'status': '400',
- }
- self.assertFalse(rule.allow(denied))
diff --git a/slog/slog_tests.py b/slog/slog_tests.py
deleted file mode 100644
index cc18f15..0000000
--- a/slog/slog_tests.py
+++ /dev/null
@@ -1,258 +0,0 @@
-# slog_tests.py - unit tests for structured logging
-#
-# Copyright 2016 QvarnLabs Ab
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import glob
-import json
-import os
-import shutil
-import tempfile
-import time
-import unittest
-
-import slog
-
-
-class StructuredLogTests(unittest.TestCase):
-
- def setUp(self):
- self.tempdir = tempfile.mkdtemp()
-
- def tearDown(self):
- shutil.rmtree(self.tempdir)
-
- def create_structured_log(self):
- filename = os.path.join(self.tempdir, 'slog.log')
- writer = slog.FileSlogWriter()
- writer.set_filename(filename)
-
- log = slog.StructuredLog()
- log.add_log_writer(writer, slog.FilterAllow())
- return log, writer, filename
-
- def read_log_entries(self, writer):
- filename = writer.get_filename()
- with open(filename) as f:
- return [json.loads(line) for line in f]
-
- def test_logs_in_json(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg', foo='foo', bar='bar', number=12765)
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- obj = objs[0]
- self.assertEqual(obj['msg_type'], 'testmsg')
- self.assertEqual(obj['foo'], 'foo')
- self.assertEqual(obj['bar'], 'bar')
- self.assertEqual(obj['number'], 12765)
-
- def test_logs_two_lines_in_json(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg1', foo='foo')
- log.log('testmsg2', bar='bar')
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 2)
- obj1, obj2 = objs
-
- self.assertEqual(obj1['msg_type'], 'testmsg1')
- self.assertEqual(obj1['foo'], 'foo')
-
- self.assertEqual(obj2['msg_type'], 'testmsg2')
- self.assertEqual(obj2['bar'], 'bar')
-
- def test_adds_some_extra_fields(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg')
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- obj = objs[0]
- self.assertEqual(obj['msg_type'], 'testmsg')
- self.assertIn('_timestamp', obj)
- self.assertIn('_process_id', obj)
- self.assertIn('_thread_id', obj)
- self.assertEqual(obj['_context'], None)
-
- def test_adds_context_when_given(self):
- log, writer, _ = self.create_structured_log()
- log.set_context('request 123')
- log.log('testmsg')
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- obj = objs[0]
- self.assertEqual(obj['_context'], 'request 123')
-
- def test_resets_context_when_requested(self):
- log, writer, _ = self.create_structured_log()
- log.set_context('request 123')
- log.reset_context()
- log.log('testmsg')
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- obj = objs[0]
- self.assertEqual(obj['_context'], None)
-
- def test_counts_messages(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg')
- log.log('testmsg')
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 2)
- self.assertEqual(objs[0]['_msg_number'], 1)
- self.assertEqual(objs[1]['_msg_number'], 2)
-
- def test_logs_traceback(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg', exc_info=True)
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- self.assertIn('_traceback', objs[0])
-
- def test_logs_unicode(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg', text='foo')
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- self.assertEqual(objs[0]['text'], 'foo')
-
- def test_logs_nonutf8(self):
- log, writer, _ = self.create_structured_log()
- notutf8 = '\x86'
- log.log('blobmsg', notutf8=notutf8)
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- self.assertEqual(objs[0]['notutf8'], notutf8)
-
- def test_logs_list(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg', items=[1, 2, 3])
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- self.assertEqual(objs[0]['items'], [1, 2, 3])
-
- def test_logs_tuple(self):
- log, writer, _ = self.create_structured_log()
- log.log('testmsg', t=(1, 2, 3))
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- # Tuples get returned as s list.
- self.assertEqual(objs[0]['t'], [1, 2, 3])
-
- def test_logs_dict(self):
- log, writer, _ = self.create_structured_log()
- dikt = {
- 'foo': 'bar',
- 'yo': [1, 2, 3],
- }
- log.log('testmsg', dikt=dikt)
- log.close()
-
- objs = self.read_log_entries(writer)
- self.assertEqual(len(objs), 1)
- self.assertEqual(objs[0]['dikt'], dikt)
-
- def test_logs_to_two_files(self):
- filename1 = os.path.join(self.tempdir, 'slog1XS')
- writer1 = slog.FileSlogWriter()
- writer1.set_filename(filename1)
-
- filename2 = os.path.join(self.tempdir, 'slog2')
- writer2 = slog.FileSlogWriter()
- writer2.set_filename(filename2)
-
- log = slog.StructuredLog()
- log.add_log_writer(writer1, slog.FilterAllow())
- log.add_log_writer(writer2, slog.FilterAllow())
-
- log.log('test', msg_text='hello')
- objs1 = self.read_log_entries(writer1)
- objs2 = self.read_log_entries(writer2)
-
- self.assertEqual(objs1, objs2)
-
-
-class FileSlogWriterTests(unittest.TestCase):
-
- def setUp(self):
- self.tempdir = tempfile.mkdtemp()
- self.filename = os.path.join(self.tempdir, 'foo.log')
-
- def tearDown(self):
- shutil.rmtree(self.tempdir)
-
- def test_gets_initial_filename_right(self):
- fw = slog.FileSlogWriter()
- fw.set_filename(self.filename)
- self.assertEqual(fw.get_filename(), self.filename)
-
- def test_gets_rotated_filename_right(self):
- fw = slog.FileSlogWriter()
- fw.set_filename(self.filename)
-
- now = time.struct_time((1969, 9, 1, 14, 30, 42, 0, 243, 0))
- self.assertEqual(
- fw.get_rotated_filename(now=now),
- os.path.join(self.tempdir, 'foo-19690901T143042.log')
- )
-
- def test_creates_file(self):
- fw = slog.FileSlogWriter()
- filename = os.path.join(self.tempdir, 'slog.log')
- fw.set_filename(filename)
- self.assertTrue(os.path.exists(filename))
-
- def test_rotates_after_size_limit(self):
- fw = slog.FileSlogWriter()
- filename = os.path.join(self.tempdir, 'slog.log')
- fw.set_filename(filename)
- fw.set_max_file_size(1)
- fw.write({'foo': 'bar'})
- filenames = glob.glob(self.tempdir + '/*.log')
- self.assertEqual(len(filenames), 2)
- self.assertTrue(filename in filenames)
-
- rotated_filename = [x for x in filenames if x != filename][0]
- objs1 = self.load_log_objs(filename)
- objs2 = self.load_log_objs(rotated_filename)
-
- self.assertEqual(len(objs1), 0)
- self.assertEqual(len(objs2), 1)
-
- def load_log_objs(self, filename):
- with open(filename) as f:
- return [json.loads(line) for line in f]