From f097c61b1c1435d4849a33930cb75332dc7158dc Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 25 Dec 2015 13:06:49 +0100 Subject: Rewrite obbench, adding yarns and Debian packaging --- obbenchlib/__init__.py | 3 + obbenchlib/benchmarker.py | 172 ++++++++++++++++++++++++++++ obbenchlib/htmlgen.py | 232 ++++++++++++++++++++++++++++++++++++++ obbenchlib/obbench.css | 27 +++++ obbenchlib/result.py | 57 ++++++++++ obbenchlib/templates/benchmark.j2 | 38 +++++++ obbenchlib/templates/index.j2 | 28 +++++ 7 files changed, 557 insertions(+) create mode 100644 obbenchlib/__init__.py create mode 100644 obbenchlib/benchmarker.py create mode 100644 obbenchlib/htmlgen.py create mode 100644 obbenchlib/obbench.css create mode 100644 obbenchlib/result.py create mode 100644 obbenchlib/templates/benchmark.j2 create mode 100644 obbenchlib/templates/index.j2 (limited to 'obbenchlib') diff --git a/obbenchlib/__init__.py b/obbenchlib/__init__.py new file mode 100644 index 0000000..9e52759 --- /dev/null +++ b/obbenchlib/__init__.py @@ -0,0 +1,3 @@ +from .benchmarker import Benchmarker +from .result import Result +from .htmlgen import HtmlGenerator diff --git a/obbenchlib/benchmarker.py b/obbenchlib/benchmarker.py new file mode 100644 index 0000000..cba86a3 --- /dev/null +++ b/obbenchlib/benchmarker.py @@ -0,0 +1,172 @@ +# Copyright 2015 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import os +import pstats +import shutil +import StringIO +import tempfile +import time + +import cliapp + +import obbenchlib + + +class Benchmarker(object): + + profile_name = 'obnam.prof' + + def __init__(self): + self.statedir = None + self.gitdir = None + self.resultdir = None + self._livedir = None + self._repodir = None + self._srcdir = None + self._config = None + self._restored = None + self._timestamp = None + self.spec = None + + def run_benchmarks(self, ref): + tempdir = self.create_temp_dir() + self._livedir = self.create_subdir(tempdir, 'live') + self._repodir = self.create_subdir(tempdir, 'repo') + self._srcdir = self.create_subdir(tempdir, 'src') + self._restored = self.create_subdir(tempdir, 'restored') + self._config = self.prepare_obnam_config(tempdir) + self._timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + + self.prepare_obnam(ref) + if not os.path.exists(self.resultdir): + os.mkdir(self.resultdir) + for benchmark in self.spec['benchmarks']: + result = self.run_benchmark(benchmark) + result.save_in_dir(self.resultdir) + self.remove_temp_dir(tempdir) + + def create_temp_dir(self): + return tempfile.mkdtemp() + + def create_subdir(self, parent, child): + pathname = os.path.join(parent, child) + os.mkdir(pathname) + return pathname + + def remove_temp_dir(self, tempdir): + shutil.rmtree(tempdir) + + def prepare_obnam_config(self, tempdir): + config = os.path.join(tempdir, 'obnam.conf') + with open(config, 'w') as f: + f.write('[config]\n') + f.write('quiet = yes\n') + f.write('repository = %s\n' % self._repodir) + f.write('root = %s\n' % self._livedir) + return config + + def prepare_obnam(self, ref): + cliapp.runcmd(['git', 'clone', self.gitdir, self._srcdir]) + cliapp.runcmd(['git', 'checkout', ref], cwd=self._srcdir) + cliapp.runcmd( + ['python', 'setup.py', 'build_ext', '-i'], + cwd=self._srcdir) + + def run_benchmark(self, benchmark): + result = obbenchlib.Result() + result.benchmark_name = benchmark['name'] + result.run_timestamp = self._timestamp + result.commit_date = self.get_commit_date() + result.commit_timestamp = self.get_commit_timestamp() + result.commit_id = self.get_commit_id() + for step in benchmark['steps']: + result.start_step() + self.run_step(result, step) + return result + + def get_commit_date(self): + timestamp = self.get_commit_timestamp() + return timestamp.split()[0] + + def get_commit_timestamp(self): + output = cliapp.runcmd( + ['git', 'show', '--date=iso', 'HEAD'], + cwd=self._srcdir) + for line in output.splitlines(): + if line.startswith('Date:'): + return line[len('Date:'):].strip() + raise Exception('commit has no Date:') + + def get_commit_id(self): + output = cliapp.runcmd(['git', 'rev-parse', 'HEAD'], cwd=self._srcdir) + return output.strip() + + def run_step(self, result, step): + if 'live' in step: + self.run_step_live(result, step['live']) + self.run_step_obnam(result, step['obnam']) + + def run_step_live(self, result, shell_command): + started = time.time() + cliapp.runcmd(['sh', '-euc', shell_command], cwd=self._livedir) + duration = time.time() - started + result.set_value('live', 'duration', duration) + + def run_step_obnam(self, result, obnam_subcommand): + funcs = { + 'backup': self.run_obnam_backup, + 'restore': self.run_obnam_restore, + } + started = time.time() + funcs[obnam_subcommand]() + duration = time.time() - started + result.set_value(obnam_subcommand, 'duration', duration) + result.set_value(obnam_subcommand, 'profile', self.read_profile()) + result.set_value( + obnam_subcommand, 'profile-text', self.read_profile_text()) + + def run_obnam_backup(self): + self.run_obnam(['backup']) + + def run_obnam_restore(self): + self.run_obnam(['restore', '-to', self._restored]) + + def run_obnam(self, args): + env = dict(os.environ) + env['OBNAM_PROFILE'] = self.profile_name + opts = ['--no-default-config', '--config', self._config] + cliapp.runcmd( + ['./obnam'] + opts + args, + env=env, + cwd=self._srcdir) + + def read_profile(self): + filename = os.path.join(self._srcdir, self.profile_name) + with open(filename) as f: + return f.read() + + def read_profile_text(self): + f = StringIO.StringIO() + filename = os.path.join(self._srcdir, self.profile_name) + p = pstats.Stats(filename, stream=f) + p.strip_dirs() + p.sort_stats('cumulative') + p.print_stats() + p.print_callees() + return f.getvalue() diff --git a/obbenchlib/htmlgen.py b/obbenchlib/htmlgen.py new file mode 100644 index 0000000..195f2c1 --- /dev/null +++ b/obbenchlib/htmlgen.py @@ -0,0 +1,232 @@ +# Copyright 2015 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import glob +import os + +import jinja2 +import markdown +import yaml + +import obbenchlib + + +class HtmlGenerator(object): + + def __init__(self): + self.statedir = None + self.resultdir = None + self.spec = None + + def generate_html(self): + results = self.load_results() + + env = jinja2.Environment( + loader=jinja2.PackageLoader('obbenchlib'), + autoescape=lambda foo: True, + extensions=['jinja2.ext.autoescape']) + + self.create_html_dir() + page_classes = [FrontPage, BenchmarkPage, ProfileData, CssFile] + for page_class in page_classes: + page = page_class() + page.env = env + page.results = results + page.spec = self.spec + + for filename, data in page.generate(): + self.write_file(filename, data) + + @property + def htmldir(self): + return os.path.join(self.statedir, 'html') + + def create_html_dir(self): + if not os.path.exists(self.htmldir): + os.mkdir(self.htmldir) + + def load_results(self): + results = [] + for filename in glob.glob(os.path.join(self.resultdir, '*.yaml')): + with open(filename) as f: + results.append(yaml.safe_load(f)) + return results + + def write_file(self, relative_path, text): + filename = os.path.join(self.htmldir, relative_path) + with open(filename, 'w') as f: + f.write(text) + + +class HtmlPage(object): + + def __init__(self): + self.env = None + self.results = None + self.spec = None + + def format_markdown(self, text): + return markdown.markdown(text) + + def get_step_names(self, benchmark): + return [step['obnam'] for step in benchmark['steps']] + + def generate(self): + raise NotImplementedError() + + def render(self, template_name, variables): + template = self.env.get_template(template_name) + return template.render(**variables) + + +class FrontPage(HtmlPage): + + def generate(self): + variables = { + 'description': self.format_markdown(self.spec['description']), + 'benchmark_names': [ + benchmark['name'] + for benchmark in sorted(self.spec['benchmarks']) + ], + 'results_table': self.results_table(), + } + yield 'index.html', self.render('index.j2', variables) + + def results_table(self): + table = {} + for result in self.results: + key = '{commit_timestamp} {commit_id} {run_timestamp}'.format( + **result) + if key not in table: + table[key] = { + 'commit_id': result['commit_id'], + 'commit_date': result['commit_date'], + } + table[key][result['benchmark_name']] = self.duration(result) + + return [table[key] for key in sorted(table.keys())] + + def duration(self, result): + total = 0 + for step in result['steps']: + for key in step: + if key != 'live': + total += step[key].get('duration', 0) + return total + + +class BenchmarkPage(HtmlPage): + + def generate(self): + benchmark_names = [ + benchmark['name'] + for benchmark in self.spec['benchmarks'] + ] + + for benchmark_name in benchmark_names: + yield self.generate_benchmark_page(benchmark_name) + + def generate_benchmark_page(self, benchmark_name): + benchmark = self.find_benchmark(benchmark_name) + table_rows = self.table_rows(benchmark) + + variables = { + 'benchmark_name': benchmark_name, + 'description': self.format_markdown( + benchmark.get('description', '')), + 'table_rows': table_rows, + 'step_names': self.get_step_names(benchmark), + } + + return ( + '{}.html'.format(benchmark_name), + self.render('benchmark.j2', variables) + ) + + def find_benchmark(self, benchmark_name): + for benchmark in self.spec['benchmarks']: + if benchmark['name'] == benchmark_name: + return benchmark + return {} + + def table_rows(self, benchmark): + results = self.get_results_for_benchmark(benchmark) + step_names = self.get_step_names(benchmark) + rows = [] + for result in results: + rows.append(self.table_row(result, step_names)) + return sorted(rows, key=lambda row: row['commit_timestamp']) + + def get_results_for_benchmark(self, benchmark): + return [ + result + for result in self.results + if result['benchmark_name'] == benchmark['name'] + ] + + def table_row(self, result, step_names): + row = { + 'result_id': result['result_id'], + 'commit_timestamp': result['commit_timestamp'], + 'commit_date': result['commit_date'], + 'commit_id': result['commit_id'], + 'total': 0, + 'steps': [], + } + for i, step in enumerate(result['steps']): + for step_name in step_names: + if step_name in step: + row['steps'].append({ + 'filename_txt': '{}_{}.txt'.format( + result['result_id'], i), + 'duration': step[step_name]['duration'], + }) + row['total'] += row['steps'][-1]['duration'] + break + return row + + +class ProfileData(HtmlPage): + + def generate(self): + for result in self.results: + for i, step in enumerate(result['steps']): + for operation in step: + if 'profile' in step[operation]: + yield self.generate_profile_data( + result, step, i, operation) + yield self.generate_profile_text( + result, step, i, operation) + + def generate_profile_data(self, result, step, i, operation): + filename = '{}_{}.prof'.format(result['result_id'], i) + return filename, step[operation]['profile'] + + def generate_profile_text(self, result, step, i, operation): + filename = '{}_{}.txt'.format(result['result_id'], i) + return filename, step[operation]['profile-text'] + + +class CssFile(object): + + def generate(self): + filename = os.path.join( + os.path.dirname(obbenchlib.__file__), 'obbench.css') + with open(filename) as f: + data = f.read() + yield 'obbench.css', data diff --git a/obbenchlib/obbench.css b/obbenchlib/obbench.css new file mode 100644 index 0000000..4e35607 --- /dev/null +++ b/obbenchlib/obbench.css @@ -0,0 +1,27 @@ +table { + border: 0; +} +th { + border-bottom: solid 1px; +} +th, td { + font-family: monospace; + text-align: left; + vertical-align: top; + padding-right: 2em; +} +td { + padding-bottom: 1em; +} +th.duration, td.duration { + width: 15em; +} +td.date { +// min-width: 30em; +} +td.commitid { +// width: 7em; +} +td.commitmsg { +// max-width: 30em; +} diff --git a/obbenchlib/result.py b/obbenchlib/result.py new file mode 100644 index 0000000..8ac24b8 --- /dev/null +++ b/obbenchlib/result.py @@ -0,0 +1,57 @@ +# Copyright 2015 Lars Wirzenius +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# =*= License: GPL-3+ =*= + + +import os +import random + +import yaml + + +class Result(object): + + def __init__(self): + self.benchmark_name = None + self.run_timestamp = None + self.commit_date = None + self.commit_timestamp = None + self.commit_id = None + self._result_id = random.randint(0, 2**64-1) + self._steps = [] + + def start_step(self): + self._steps.append({}) + + def set_value(self, operation, kind, value): + step = self._steps[-1] + if operation not in step: + step[operation] = {} + step[operation][kind] = value + + def save_in_dir(self, dirname): + o = { + 'result_id': self._result_id, + 'benchmark_name': self.benchmark_name, + 'run_timestamp': self.run_timestamp, + 'commit_date': self.commit_date, + 'commit_timestamp': self.commit_timestamp, + 'commit_id': self.commit_id, + 'steps': self._steps, + } + filename = os.path.join(dirname, '{}.yaml'.format(self._result_id)) + with open(filename, 'w') as f: + yaml.safe_dump(o, stream=f) diff --git a/obbenchlib/templates/benchmark.j2 b/obbenchlib/templates/benchmark.j2 new file mode 100644 index 0000000..1a723a1 --- /dev/null +++ b/obbenchlib/templates/benchmark.j2 @@ -0,0 +1,38 @@ + + + Obnam benchmark: {{ benchmark_name }} + + + +

Obnam benchmark: {{ benchmark_name }}

+ +

Front page

+ + {{ description|safe }} + + + + + + {% for step_name in step_names %} + + {% endfor %} + + + {% for row in table_rows %} + + + + {% for step in row.steps %} + + {% endfor %} + + + {% endfor %} +
datecommit{{ step_name }}total
{{ row.commit_date }}{{ '%.7s'|format(row.commit_id) }} + + {{ '%.1f'|format(step.duration) }} + + {{ '%.1f'|format(row.total) }}
+ + diff --git a/obbenchlib/templates/index.j2 b/obbenchlib/templates/index.j2 new file mode 100644 index 0000000..3212abb --- /dev/null +++ b/obbenchlib/templates/index.j2 @@ -0,0 +1,28 @@ + + + Obnam benchmarks + + + +

Obnam benchmarks

+ {{ description|safe }} + + + + + {% for name in benchmark_names %} + + {% endfor %} + + {% for row in results_table %} + + + + {% for name in benchmark_names %} + + {% endfor %} + + {% endfor %} +
datecommit{{ name }}
{{ row.commit_date }}{{ '%.7s'|format(row.commit_id) }}{{ '%.1f'|format(row[name]) }}
+ + -- cgit v1.2.1