#!/usr/bin/env python3 # Copyright (C) 2019 Lars Wirzenius # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . import copy import json import logging import os import sys import time import requests MAX_SNIPPETS = 1000 def generate_snippet_texts(n): for i in range(n): yield 'log line {}\n'.format(i) def full_log(n): return ''.join(generate_snippet_texts(n)) def store_object(url, token, obj): url = '{}/res'.format(url) headers = { 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/json', } r = requests.post(url, headers=headers, data=json.dumps(obj)) assert r.ok def get_object(url, token, rid): url = '{}/res'.format(url) headers = { 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/json', 'Muck-Id': rid, } r = requests.get(url, headers=headers) assert r.ok return r.json() def search_objects_of_type(url, token, type_name): search = '{}/search'.format(url) body = { 'cond': [ { "where": "data", "op": "==", "field": "_type", "pattern": type_name, }, ], } headers = { 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/json', } r = requests.get(search, headers=headers, data=json.dumps(body)) assert r.ok obj = r.json() return obj['resources'] def delete_object(url, token, rid): url = '{}/res'.format(url) headers = { 'Authorization': 'Bearer {}'.format(token), 'Muck-Id': rid, } r = requests.delete(url, headers=headers) assert r.ok def time_to_combine(seq): # seq numbers start at 0 num = seq + 1 return (num % MAX_SNIPPETS) == 0 def combine_snippets(url, token): snippets = [] first_seq = None rids = search_objects_of_type(url, token, 'snippet') if not rids: return first = get_object(url, token, rids[0]) combined = { '_type': 'combined-snippet', 'seq': first['seq'], 'text': assemble_from_objects(url, token, rids), } store_object(url, token, combined) for rid in rids: delete_object(url, token, rid) def create_snippets(url, token, n): for i, snippet in enumerate(generate_snippet_texts(n)): obj = { '_type': 'snippet', 'seq': i, 'text': snippet, } store_object(url, token, obj) if time_to_combine(i): combine_snippets(url, token) def assemble_from_objects(url, token, ids): objs = [get_object(url, token, rid) for rid in ids] objs.sort(key=lambda o: o['seq']) return ''.join(o['text'] for o in objs) def get_full_log(url, token, combined_ids, snippet_ids): combined = assemble_from_objects(url, token, combined_ids) snippets = assemble_from_objects(url, token, snippet_ids) return combined + snippets def measure(func): started = time.time() ret = func() now = time.time() return now - started, ret url = sys.argv[1] token = sys.argv[2] N = int(sys.argv[3]) print('creating snippets') creation_secs, _ = measure(lambda: create_snippets(url, token, N)) print('getting list of combined snippets') get_combined_secs, combined_ids = measure( lambda: search_objects_of_type(url, token, 'combined-snippet')) print('combined ids', len(combined_ids)) print('getting list of snippets') get_snippets_secs, snippet_ids = measure( lambda: search_objects_of_type(url, token, 'snippet')) print('snippet ids', len(snippet_ids)) print('reconstructing full log') get_log_secs, log = measure( lambda: get_full_log(url, token, combined_ids, snippet_ids)) expected = full_log(N) if expected != log: print('ERROR: actual log differs from expected') print('actual:') print(repr(log)) sys.exit(1) print('OK') print('%.0f' % creation_secs, 'creation time') print('%.0f' % get_combined_secs, 'list combined snippets') print('%.0f' % get_snippets_secs, 'list snippets') print('%.0f' % get_log_secs, 'assemble log')