#!/usr/bin/python
# Copyright 2015 Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
# =*= License: GPL-3+ =*=
import glob
import os
import shutil
import sys
import tempfile
import time
import cliapp
import yaml
class ObnamBenchmarker(cliapp.Application):
def process_args(self, args):
if not args:
raise cliapp.AppException('Need benchmark spec filename')
spec = self.read_benchmark_spec(args[0])
self.logger = IndentedLogger()
tempdir = tempfile.mkdtemp()
for treeish in args[1:]:
self.logger.msg('Benchmarking treeish %s' % treeish)
with self.logger:
self.run_all_benchmarks(spec, treeish, tempdir)
self.logger.msg('Generating HTML')
self.generate_html(spec)
self.logger.msg('Cleaning up')
shutil.rmtree(tempdir)
def read_benchmark_spec(self, filename):
with open(filename) as f:
return yaml.safe_load(f)
def run_all_benchmarks(self, spec, treeish, tempdir):
checkout = self.get_treeish(spec, treeish, tempdir)
self.prepare_obnam(checkout)
for benchmark in spec.get('benchmarks', []):
result = self.run_one_benchmark(spec, benchmark, tempdir, checkout)
self.save_result(spec, benchmark, result)
def get_treeish(self, spec, treeish, tempdir):
checkout = os.path.join(tempdir, 'git')
if not os.path.exists(checkout):
cliapp.runcmd(['git', 'clone', spec['git'], checkout])
cliapp.runcmd(['git', 'checkout', treeish], cwd=checkout)
cliapp.runcmd(['git', 'clean', '-fdxq'], cwd=checkout)
return checkout
def prepare_obnam(self, checkout):
cliapp.runcmd(['python', 'setup.py', 'build_ext', '-i'], cwd=checkout)
def run_one_benchmark(self, spec, benchmark, tempdir, checkout):
self.logger.msg('Running benchmark %s' % benchmark['name'])
with self.logger:
result = BenchmarkResult()
result.collect_info_from_spec(benchmark)
result.collect_info_from_checkout(checkout)
config = self.create_obnam_config(spec, benchmark, tempdir)
live = self.create_live_dir(tempdir)
for step in benchmark.get('steps', []):
self.run_benchmark_step(
step, tempdir, checkout, config, live, result)
return result
def create_obnam_config(self, spec, benchmark, tempdir):
config = os.path.join(tempdir, 'obnam.conf')
with open(config, 'w') as f:
f.write('[config]\n')
f.write('repository = %s\n' % os.path.join(tempdir, 'repo'))
f.write('root = %s\n' % self.get_live_data(tempdir))
for key, value in spec.get('obnam_config', {}).items():
f.write('%s = %s\n' % (key, value))
for key, value in benchmark.get('obnam_config', {}).items():
f.write('%s = %s\n' % (key, value))
return config
def get_live_data(self, tempdir):
return os.path.join(tempdir, 'live')
def create_live_dir(self, tempdir):
live = self.get_live_data(tempdir)
if os.path.exists(live):
shutil.rmtree(live)
os.mkdir(live)
return live
def run_benchmark_step(self,
step, tempdir, checkout, config, live, result):
step_info = dict(step)
if 'live' in step:
self.logger.msg('Creating live data: %s' % step['live'])
cliapp.runcmd(['sh', '-euc', step['live']], cwd=live)
action = step['obnam']
self.logger.msg('Obnam %s' % action)
func = funcs = {
'backup': self.run_backup,
'restore': self.run_restore,
}
started = time.time()
funcs[action](tempdir, checkout, config)
ended = time.time()
step_info['duration'] = ended - started
result.add_step(step_info)
def run_backup(self, tempdir, checkout, config):
self.run_obnam(checkout, ['backup', '--config', config])
def run_restore(self, tempdir, checkout, config):
restored = os.path.join(tempdir, 'restored')
if os.path.exists(restored):
shutil.rmtree(restored)
self.run_obnam(
checkout,
['restore', '--config', config, '--to', restored])
def run_obnam(self, checkout, args):
cliapp.runcmd(
['./obnam', '--no-default-config'] + args,
cwd=checkout)
def save_result(self, spec, benchmark, result):
obj = result.as_dict()
pathname = self.get_report_pathname(spec, benchmark, result)
with open(pathname, 'w') as f:
yaml.safe_dump(obj, stream=f, default_flow_style=False, indent=4)
def get_report_pathname(self, spec, benchmark, result):
return os.path.join(
spec['reports_dir'],
'%s_%s.yaml' % (result.get_commit_id(), benchmark['name']))
def generate_html(self, spec):
objs = self.read_results_files(spec)
for obj in objs:
self.write_benchmark_page(spec, obj)
self.write_summary_page(spec, objs)
self.copy_css_file(spec)
def read_results_files(self, spec):
objs = []
for filename in glob.glob(os.path.join(spec['reports_dir'], '*.yaml')):
with open(filename) as f:
objs.append(yaml.safe_load(f))
return objs
def write_benchmark_page(self, spec, obj):
steps = [s for s in obj['steps'] if 'obnam' in s]
filename = os.path.join(
spec['html_dir'],
'{}_{}.html'.format(obj['commit_id'], obj['name']))
with open(filename, 'w') as f:
f.write('\n')
f.write('
\n')
f.write('\n')
f.write('\n')
def q(self, text):
'''Quote for HTML'''
text = str(text)
text = '&'.join(text.split('&'))
text = '<'.join(text.split('<'))
text = '>'.join(text.split('>'))
return text
def write_summary_page(self, spec, objs):
benchmark_names = self.find_benchmark_names(objs)
runs = self.create_table_of_benchmark_runs(benchmark_names, objs)
filename = os.path.join(spec['html_dir'], 'index.html')
with open(filename, 'w') as f:
f.write('\n')
f.write('\n')
f.write('Obnam benchmark: summary\n')
f.write('\n')
f.write('\n')
f.write('\n')
f.write('
Obnam benchmark: summary
\n')
f.write('
\n')
f.write('
\n')
f.write('
date
\n')
f.write('
commit
\n')
for name in benchmark_names:
f.write('
{name}
\n'.format(name=self.q(name)))
f.write('
\n')
for run in runs:
f.write('
\n')
f.write('
{date}
\n'.format(date=self.q(run['date'])))
f.write(
'
{commit}
\n'.format(
commit=self.q(run['commit_id'])))
for name in benchmark_names:
link = '{commit}_{name}.html'.format(
commit=self.q(run['commit_id']),
name=self.q(name))
duration = '%.1f' % run['durations'][name]
f.write(
'