Rewrite obbench, adding yarns and Debian packaging

7 files changed, 557 insertions, 0 deletions

diff --git a/obbenchlib/__init__.py b/obbenchlib/__init__.py
new file mode 100644
index 0000000..9e52759
--- /dev/null
+++ b/obbenchlib/__init__.py
@@ -0,0 +1,3 @@
+from .benchmarker import Benchmarker
+from .result import Result
+from .htmlgen import HtmlGenerator
diff --git a/obbenchlib/benchmarker.py b/obbenchlib/benchmarker.py
new file mode 100644
index 0000000..cba86a3
--- /dev/null
+++ b/obbenchlib/benchmarker.py
@@ -0,0 +1,172 @@
+# Copyright 2015  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import os
+import pstats
+import shutil
+import StringIO
+import tempfile
+import time
+
+import cliapp
+
+import obbenchlib
+
+
+class Benchmarker(object):
+
+    profile_name = 'obnam.prof'
+
+    def __init__(self):
+        self.statedir = None
+        self.gitdir = None
+        self.resultdir = None
+        self._livedir = None
+        self._repodir = None
+        self._srcdir = None
+        self._config = None
+        self._restored = None
+        self._timestamp = None
+        self.spec = None
+
+    def run_benchmarks(self, ref):
+        tempdir = self.create_temp_dir()
+        self._livedir = self.create_subdir(tempdir, 'live')
+        self._repodir = self.create_subdir(tempdir, 'repo')
+        self._srcdir = self.create_subdir(tempdir, 'src')
+        self._restored = self.create_subdir(tempdir, 'restored')
+        self._config = self.prepare_obnam_config(tempdir)
+        self._timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
+
+        self.prepare_obnam(ref)
+        if not os.path.exists(self.resultdir):
+            os.mkdir(self.resultdir)
+        for benchmark in self.spec['benchmarks']:
+            result = self.run_benchmark(benchmark)
+            result.save_in_dir(self.resultdir)
+        self.remove_temp_dir(tempdir)
+
+    def create_temp_dir(self):
+        return tempfile.mkdtemp()
+
+    def create_subdir(self, parent, child):
+        pathname = os.path.join(parent, child)
+        os.mkdir(pathname)
+        return pathname
+
+    def remove_temp_dir(self, tempdir):
+        shutil.rmtree(tempdir)
+
+    def prepare_obnam_config(self, tempdir):
+        config = os.path.join(tempdir, 'obnam.conf')
+        with open(config, 'w') as f:
+            f.write('[config]\n')
+            f.write('quiet = yes\n')
+            f.write('repository = %s\n' % self._repodir)
+            f.write('root = %s\n' % self._livedir)
+        return config
+
+    def prepare_obnam(self, ref):
+        cliapp.runcmd(['git', 'clone', self.gitdir, self._srcdir])
+        cliapp.runcmd(['git', 'checkout', ref], cwd=self._srcdir)
+        cliapp.runcmd(
+            ['python', 'setup.py', 'build_ext', '-i'],
+            cwd=self._srcdir)
+
+    def run_benchmark(self, benchmark):
+        result = obbenchlib.Result()
+        result.benchmark_name = benchmark['name']
+        result.run_timestamp = self._timestamp
+        result.commit_date = self.get_commit_date()
+        result.commit_timestamp = self.get_commit_timestamp()
+        result.commit_id = self.get_commit_id()
+        for step in benchmark['steps']:
+            result.start_step()
+            self.run_step(result, step)
+        return result
+
+    def get_commit_date(self):
+        timestamp = self.get_commit_timestamp()
+        return timestamp.split()[0]
+
+    def get_commit_timestamp(self):
+        output = cliapp.runcmd(
+            ['git', 'show', '--date=iso', 'HEAD'],
+            cwd=self._srcdir)
+        for line in output.splitlines():
+            if line.startswith('Date:'):
+                return line[len('Date:'):].strip()
+        raise Exception('commit has no Date:')
+
+    def get_commit_id(self):
+        output = cliapp.runcmd(['git', 'rev-parse', 'HEAD'], cwd=self._srcdir)
+        return output.strip()
+
+    def run_step(self, result, step):
+        if 'live' in step:
+            self.run_step_live(result, step['live'])
+        self.run_step_obnam(result, step['obnam'])
+
+    def run_step_live(self, result, shell_command):
+        started = time.time()
+        cliapp.runcmd(['sh', '-euc', shell_command], cwd=self._livedir)
+        duration = time.time() - started
+        result.set_value('live', 'duration', duration)
+
+    def run_step_obnam(self, result, obnam_subcommand):
+        funcs = {
+            'backup': self.run_obnam_backup,
+            'restore': self.run_obnam_restore,
+        }
+        started = time.time()
+        funcs[obnam_subcommand]()
+        duration = time.time() - started
+        result.set_value(obnam_subcommand, 'duration', duration)
+        result.set_value(obnam_subcommand, 'profile', self.read_profile())
+        result.set_value(
+            obnam_subcommand, 'profile-text', self.read_profile_text())
+
+    def run_obnam_backup(self):
+        self.run_obnam(['backup'])
+
+    def run_obnam_restore(self):
+        self.run_obnam(['restore', '-to', self._restored])
+
+    def run_obnam(self, args):
+        env = dict(os.environ)
+        env['OBNAM_PROFILE'] = self.profile_name
+        opts = ['--no-default-config', '--config', self._config]
+        cliapp.runcmd(
+            ['./obnam'] + opts + args,
+            env=env,
+            cwd=self._srcdir)
+
+    def read_profile(self):
+        filename = os.path.join(self._srcdir, self.profile_name)
+        with open(filename) as f:
+            return f.read()
+
+    def read_profile_text(self):
+        f = StringIO.StringIO()
+        filename = os.path.join(self._srcdir, self.profile_name)
+        p = pstats.Stats(filename, stream=f)
+        p.strip_dirs()
+        p.sort_stats('cumulative')
+        p.print_stats()
+        p.print_callees()
+        return f.getvalue()
diff --git a/obbenchlib/htmlgen.py b/obbenchlib/htmlgen.py
new file mode 100644
index 0000000..195f2c1
--- /dev/null
+++ b/obbenchlib/htmlgen.py
@@ -0,0 +1,232 @@
+# Copyright 2015  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import glob
+import os
+
+import jinja2
+import markdown
+import yaml
+
+import obbenchlib
+
+
+class HtmlGenerator(object):
+
+    def __init__(self):
+        self.statedir = None
+        self.resultdir = None
+        self.spec = None
+
+    def generate_html(self):
+        results = self.load_results()
+
+        env = jinja2.Environment(
+            loader=jinja2.PackageLoader('obbenchlib'),
+            autoescape=lambda foo: True,
+            extensions=['jinja2.ext.autoescape'])
+
+        self.create_html_dir()
+        page_classes = [FrontPage, BenchmarkPage, ProfileData, CssFile]
+        for page_class in page_classes:
+            page = page_class()
+            page.env = env
+            page.results = results
+            page.spec = self.spec
+
+            for filename, data in page.generate():
+                self.write_file(filename, data)
+
+    @property
+    def htmldir(self):
+        return os.path.join(self.statedir, 'html')
+
+    def create_html_dir(self):
+        if not os.path.exists(self.htmldir):
+            os.mkdir(self.htmldir)
+
+    def load_results(self):
+        results = []
+        for filename in glob.glob(os.path.join(self.resultdir, '*.yaml')):
+            with open(filename) as f:
+                results.append(yaml.safe_load(f))
+        return results
+
+    def write_file(self, relative_path, text):
+        filename = os.path.join(self.htmldir, relative_path)
+        with open(filename, 'w') as f:
+            f.write(text)
+
+
+class HtmlPage(object):
+
+    def __init__(self):
+        self.env = None
+        self.results = None
+        self.spec = None
+
+    def format_markdown(self, text):
+        return markdown.markdown(text)
+
+    def get_step_names(self, benchmark):
+        return [step['obnam'] for step in benchmark['steps']]
+
+    def generate(self):
+        raise NotImplementedError()
+
+    def render(self, template_name, variables):
+        template = self.env.get_template(template_name)
+        return template.render(**variables)
+
+
+class FrontPage(HtmlPage):
+
+    def generate(self):
+        variables = {
+            'description': self.format_markdown(self.spec['description']),
+            'benchmark_names': [
+                benchmark['name']
+                for benchmark in sorted(self.spec['benchmarks'])
+            ],
+            'results_table': self.results_table(),
+        }
+        yield 'index.html', self.render('index.j2', variables)
+
+    def results_table(self):
+        table = {}
+        for result in self.results:
+            key = '{commit_timestamp} {commit_id} {run_timestamp}'.format(
+                **result)
+            if key not in table:
+                table[key] = {
+                    'commit_id': result['commit_id'],
+                    'commit_date': result['commit_date'],
+                }
+            table[key][result['benchmark_name']] = self.duration(result)
+
+        return [table[key] for key in sorted(table.keys())]
+
+    def duration(self, result):
+        total = 0
+        for step in result['steps']:
+            for key in step:
+                if key != 'live':
+                    total += step[key].get('duration', 0)
+        return total
+
+
+class BenchmarkPage(HtmlPage):
+
+    def generate(self):
+        benchmark_names = [
+            benchmark['name']
+            for benchmark in self.spec['benchmarks']
+        ]
+
+        for benchmark_name in benchmark_names:
+            yield self.generate_benchmark_page(benchmark_name)
+
+    def generate_benchmark_page(self, benchmark_name):
+        benchmark = self.find_benchmark(benchmark_name)
+        table_rows = self.table_rows(benchmark)
+
+        variables = {
+            'benchmark_name': benchmark_name,
+            'description': self.format_markdown(
+                benchmark.get('description', '')),
+            'table_rows': table_rows,
+            'step_names': self.get_step_names(benchmark),
+        }
+
+        return (
+            '{}.html'.format(benchmark_name),
+            self.render('benchmark.j2', variables)
+        )
+
+    def find_benchmark(self, benchmark_name):
+        for benchmark in self.spec['benchmarks']:
+            if benchmark['name'] == benchmark_name:
+                return benchmark
+        return {}
+
+    def table_rows(self, benchmark):
+        results = self.get_results_for_benchmark(benchmark)
+        step_names = self.get_step_names(benchmark)
+        rows = []
+        for result in results:
+            rows.append(self.table_row(result, step_names))
+        return sorted(rows, key=lambda row: row['commit_timestamp'])
+
+    def get_results_for_benchmark(self, benchmark):
+        return [
+            result
+            for result in self.results
+            if result['benchmark_name'] == benchmark['name']
+        ]
+
+    def table_row(self, result, step_names):
+        row = {
+            'result_id': result['result_id'],
+            'commit_timestamp': result['commit_timestamp'],
+            'commit_date': result['commit_date'],
+            'commit_id': result['commit_id'],
+            'total': 0,
+            'steps': [],
+        }
+        for i, step in enumerate(result['steps']):
+            for step_name in step_names:
+                if step_name in step:
+                    row['steps'].append({
+                        'filename_txt': '{}_{}.txt'.format(
+                            result['result_id'], i),
+                        'duration': step[step_name]['duration'],
+                    })
+                    row['total'] += row['steps'][-1]['duration']
+                    break
+        return row
+
+
+class ProfileData(HtmlPage):
+
+    def generate(self):
+        for result in self.results:
+            for i, step in enumerate(result['steps']):
+                for operation in step:
+                    if 'profile' in step[operation]:
+                        yield self.generate_profile_data(
+                            result, step, i, operation)
+                        yield self.generate_profile_text(
+                            result, step, i, operation)
+
+    def generate_profile_data(self, result, step, i, operation):
+        filename = '{}_{}.prof'.format(result['result_id'], i)
+        return filename, step[operation]['profile']
+
+    def generate_profile_text(self, result, step, i, operation):
+        filename = '{}_{}.txt'.format(result['result_id'], i)
+        return filename, step[operation]['profile-text']
+
+
+class CssFile(object):
+
+    def generate(self):
+        filename = os.path.join(
+            os.path.dirname(obbenchlib.__file__), 'obbench.css')
+        with open(filename) as f:
+            data = f.read()
+        yield 'obbench.css', data
diff --git a/obbenchlib/obbench.css b/obbenchlib/obbench.css
new file mode 100644
index 0000000..4e35607
--- /dev/null
+++ b/obbenchlib/obbench.css
@@ -0,0 +1,27 @@
+table {
+    border: 0;
+}
+th {
+    border-bottom: solid 1px;
+}
+th, td {
+    font-family: monospace;
+    text-align: left;
+    vertical-align: top;
+    padding-right: 2em;
+}
+td {
+    padding-bottom: 1em;
+}
+th.duration, td.duration {
+    width: 15em;
+}
+td.date {
+//    min-width: 30em;
+}
+td.commitid {
+//    width: 7em;
+}
+td.commitmsg {
+//    max-width: 30em;
+}
diff --git a/obbenchlib/result.py b/obbenchlib/result.py
new file mode 100644
index 0000000..8ac24b8
--- /dev/null
+++ b/obbenchlib/result.py
@@ -0,0 +1,57 @@
+# Copyright 2015  Lars Wirzenius
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-3+ =*=
+
+
+import os
+import random
+
+import yaml
+
+
+class Result(object):
+
+    def __init__(self):
+        self.benchmark_name = None
+        self.run_timestamp = None
+        self.commit_date = None
+        self.commit_timestamp = None
+        self.commit_id = None
+        self._result_id = random.randint(0, 2**64-1)
+        self._steps = []
+
+    def start_step(self):
+        self._steps.append({})
+
+    def set_value(self, operation, kind, value):
+        step = self._steps[-1]
+        if operation not in step:
+            step[operation] = {}
+        step[operation][kind] = value
+
+    def save_in_dir(self, dirname):
+        o = {
+            'result_id': self._result_id,
+            'benchmark_name': self.benchmark_name,
+            'run_timestamp': self.run_timestamp,
+            'commit_date': self.commit_date,
+            'commit_timestamp': self.commit_timestamp,
+            'commit_id': self.commit_id,
+            'steps': self._steps,
+        }
+        filename = os.path.join(dirname, '{}.yaml'.format(self._result_id))
+        with open(filename, 'w') as f:
+            yaml.safe_dump(o, stream=f)
diff --git a/obbenchlib/templates/benchmark.j2 b/obbenchlib/templates/benchmark.j2
new file mode 100644
index 0000000..1a723a1
--- /dev/null
+++ b/obbenchlib/templates/benchmark.j2
@@ -0,0 +1,38 @@
+<html>
+  <head>
+    <title>Obnam benchmark: {{ benchmark_name }}</title>
+    <link rel="stylesheet" href="obbench.css" type="text/css" />
+  </head>
+  <body>
+    <h1>Obnam benchmark: {{ benchmark_name }}</h1>
+
+    <p><a href="index.html">Front page</a></p>
+
+    {{ description|safe }}
+
+    <table>
+      <tr>
+        <th>date</th>
+        <th>commit</th>
+        {% for step_name in step_names %}
+        <th>{{ step_name }}</th>
+        {% endfor %}
+        <th>total</th>
+      </tr>
+      {% for row in table_rows %}
+      <tr>
+        <td>{{ row.commit_date }}</td>
+        <td>{{ '%.7s'|format(row.commit_id) }}</td>
+        {% for step in row.steps %}
+        <td>
+          <a href="{{ step.filename_txt }}">
+            {{ '%.1f'|format(step.duration) }}
+          </a>
+        </td>
+        {% endfor %}
+        <td>{{ '%.1f'|format(row.total) }}</td>
+      </tr>
+      {% endfor %}
+    </table>
+  </body>
+</html>
diff --git a/obbenchlib/templates/index.j2 b/obbenchlib/templates/index.j2
new file mode 100644
index 0000000..3212abb
--- /dev/null
+++ b/obbenchlib/templates/index.j2
@@ -0,0 +1,28 @@
+<html>
+  <head>
+    <title>Obnam benchmarks</title>
+    <link rel="stylesheet" href="obbench.css" type="text/css" />
+  </head>
+  <body>
+    <h1>Obnam benchmarks</h1>
+    {{ description|safe }}
+    <table>
+      <tr>
+        <th>date</th>
+        <th>commit</th>
+        {% for name in benchmark_names %}
+        <th>{{ name }}</th>
+        {% endfor %}
+      </tr>
+      {% for row in results_table %}
+      <tr>
+        <td>{{ row.commit_date }}</td>
+        <td>{{ '%.7s'|format(row.commit_id) }}</td>
+        {% for name in benchmark_names %}
+        <td><a href="{{ name }}.html">{{ '%.1f'|format(row[name]) }}</a></td>
+        {% endfor %}
+      </tr>
+      {% endfor %}
+    </table>
+  </body>
+</html>