# Copyright 2015 Lars Wirzenius # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # =*= License: GPL-3+ =*= import os import StringIO import subprocess import time import cliapp import icklib class Project(object): def __init__(self, name): self.name = name self.progress = None self.git_specs = [] self.repo_signing_key = None self.package_signing_key = None self.commands = [] self.local_commands = [] self.env = {} self.dput_targets = [] self.dput_unstable = [] self.pipelines = {} def set_progress(self, progress): self.progress = progress def add_git_spec(self, git_spec): self.git_specs.append(git_spec) def set_repo_signing_key(self, key_id): self.repo_signing_key = key_id def set_package_signing_key(self, key_id): self.package_signing_key = key_id def add_shell_command(self, command): self.commands.append(command) def add_local_command(self, command): self.local_commands.append(command) def add_dput_target(self, dput_target): self.dput_targets.append(dput_target) def add_dput_unstable(self, dput_target): self.dput_unstable.append(dput_target) def set_build_pipelines(self, pipelines): self.pipelines = pipelines def set_env(self, env): self.env = env def build(self, statedir, targets): for pipeline_name, pipeline_steps in self.pipelines.items(): pipeline = icklib.BuildPipeline(pipeline_name) pipeline.set_build_steps(pipeline_steps) run_state = icklib.RunState() run_state.pipeline = pipeline run_state.progress = self.progress run_state.logger = icklib.Logger() run_state.log_catcher = StringIO.StringIO() run_state.logger.add_output_file(run_state.log_catcher) self.run_state = run_state self.progress['pipeline'] = pipeline_name run_state.logger.important( 'Build log starts {timestamp}', timestamp=self.current_timestamp()) run_state.logger.important( 'Building project {project}', project=self.name) run_state.logger.important( 'Executing pipeline {name}', name=pipeline_name) run_state.started = time.time() try: pipeline.build(self, statedir, targets, run_state) except: if run_state.build_info is not None: run_state.build_info.status = 'FAILURE' run_state.build_info.duration = ( time.time() - run_state.started) run_state.build_info.save() run_state.logger.important('') run_state.logger.important('Build status: FAILURE') run_state.logger.important( 'Build log ends {timestamp}', timestamp=self.current_timestamp()) raise if run_state.build_info is not None: run_state.logger.important( 'Build status: {status}', status=run_state.build_info.status) run_state.logger.important( 'Build log ends {timestamp}', timestamp=self.current_timestamp()) def current_timestamp(self): timezone = '{sign}{h}{m}'.format( sign='-' if time.timezone < 0 else '+', h=abs(time.timezone) / 3600, m=(abs(time.timezone) % 3600) / 60) return '{timestamp} {timezone}'.format( timestamp=time.strftime('%Y-%m-%d %H:%M:%S'), timezone=timezone) def run_locally(self, argv, displayed_cmd, cwd): self.report_running_locally(displayed_cmd) self.run_state.logger.important( 'On local shell run {command}', command=displayed_cmd) whole_argv = self.create_argv_with_env() + argv line_logger = icklib.LineLogger(self.progress, self.run_state.logger) with self.run_state.logger: self.run_state.logger.chatty('+ {argv}', argv=' '.join(whole_argv)) cliapp.runcmd( whole_argv, stderr=subprocess.STDOUT, stdout_callback=line_logger.log_lines, cwd=cwd) line_logger.log_rest() self.report_finished_on_target() def create_argv_with_env(self): env_argv = ['env'] for name, value in self.env.items(): env_argv.append('%s=%s' % (name, value)) return env_argv def report_running_locally(self, displayed_cmd): progress = { 'target': 'local', 'command': '\\n'.join(displayed_cmd.split('\n')), } for i in range(self.run_state.progress.max_output_lines): progress['line%d' % i] = '' for key, value in progress.items(): self.run_state.progress[key] = value def run_argv_on_target(self, target, argv, displayed_cmd, remote_cwd): self.report_running_on_target(target, displayed_cmd) self.run_state.logger.important( 'On {address} run {command}', address=target.address, command=displayed_cmd) env_argv = ['env'] for name, value in self.env.items(): env_argv.append('%s=%s' % (name, value)) whole_argv = self.create_argv_with_env() + argv line_logger = icklib.LineLogger(self.progress, self.run_state.logger) with self.run_state.logger: self.run_state.logger.chatty('+ %s' % ' '.join(whole_argv)) target.ssh_runcmd( whole_argv, remote_cwd=remote_cwd, tty=False, stderr=subprocess.STDOUT, stdout_callback=line_logger.log_lines) line_logger.log_rest() self.report_finished_on_target() def report_running_on_target(self, target, displayed_cmd): progress = { 'target': target.name, 'command': '\\n'.join(displayed_cmd.split('\n')), } for i in range(self.run_state.progress.max_output_lines): progress['line%d' % i] = '' for key, value in progress.items(): self.run_state.progress[key] = value def report_finished_on_target(self): keys = ['target', 'command'] for i in range(self.run_state.progress.max_output_lines): keys.append('line%d' % i) for key in keys: self.run_state.progress[key] = '' def copy_cleanly_to_target(self, target, remote_tmp): self.run_state.logger.chatty( 'Copy helper script (./cleanly) to target') local_cleanly_path = os.path.join( os.path.dirname(icklib.__file__), 'cleanly') remote_cleanly_path = os.path.join(remote_tmp, 'cleanly') cliapp.runcmd( ['rsync', '-s', local_cleanly_path, '%s:%s' % (target.address, remote_cleanly_path)]) return remote_cleanly_path def run_cleanly(self, target, remote_git_dir, cleanly_path, args): self.run_argv_on_target( target, ['python', cleanly_path] + args, 'cleanly ' + args[-1], remote_git_dir) def create_projects_from_ick(ick, wanted_names): pipelines = { 'shell': [ icklib.LoadProjectInfo, icklib.CloneGits, icklib.FindCurrentGitCommits, icklib.FindNewCommitToBuildForShell, icklib.CreateBuildInfo, icklib.RunShellCommandsOnEachTarget, icklib.FinishBuildInfo, icklib.SaveProjectInfo, ], 'local-shell': [ icklib.LoadProjectInfo, icklib.CloneGits, icklib.FindCurrentGitCommits, icklib.FindNewCommitToBuildForLocalShell, icklib.CreateBuildInfo, icklib.RunLocalShellCommands, icklib.FinishBuildInfo, icklib.SaveProjectInfo, ], 'debian-ci': [ icklib.LoadProjectInfo, icklib.CloneGits, icklib.FindCurrentGitCommits, icklib.FindNewCommitToBuildForDebianCI, icklib.CreateBuildInfo, icklib.CollectDebianInfoAboutTargets, icklib.CreateCITarball, icklib.CreateDebianSourcePackagesForCI, icklib.CreateCIDebianBinaryPackages, icklib.FindDebianChangesFiles, icklib.SetupAPTRepository, icklib.UploadDebianPackagesToCIRepo, icklib.FinishBuildInfo, icklib.SaveProjectInfo, ], 'debian-release': [ icklib.LoadProjectInfo, icklib.CloneGits, icklib.FindCurrentGitCommits, icklib.FindTagsToBuildForDebianRelease, icklib.CreateBuildInfo, icklib.CollectDebianInfoAboutTargets, icklib.CreateReleaseTarballs, icklib.CreateDebianSourcePackagesForRelease, icklib.CreateDebianBinaryPackagesForRelease, icklib.FindDebianChangesFiles, icklib.SetupAPTRepository, icklib.UploadDebianPackagesToCIRepo, icklib.PublishDebianPackages, icklib.FinishBuildInfo, icklib.SaveProjectInfo, ], } projects = [] if not wanted_names: wanted_names = ick.get('projects', {}).keys() for project_name, project_dict in ick.get('projects', {}).items(): if project_name in wanted_names: p = Project(project_name) for git_spec in create_git_specs_for_project(project_dict): p.add_git_spec(git_spec) if 'repo-signing-key' in ick: p.set_repo_signing_key(ick['repo-signing-key']) if 'package-signing-key' in project_dict: p.set_package_signing_key(project_dict['package-signing-key']) if 'shell' in project_dict: p.add_shell_command(project_dict['shell']) if 'local-shell' in project_dict: p.add_local_command(project_dict['local-shell']) if 'debian-release' in project_dict: release = project_dict['debian-release'] dput_targets = release.get('dput-targets', []) for dput_target in dput_targets: p.add_dput_target(dput_target) dput_unstable = release.get('dput-unstable', []) for dput_target in dput_unstable: p.add_dput_unstable(dput_target) p.set_build_pipelines( dict((x, pipelines[x]) for x in project_dict['pipelines'])) p.set_env(project_dict.get('env', {})) projects.append(p) return projects def create_git_specs_for_project(project_dict): if 'git' in project_dict: yield GitSpec(project_dict['git'], project_dict['branch'], '.') elif 'gits' in project_dict: for git_dict in project_dict['gits']: yield GitSpec( git_dict['git'], git_dict['branch'], git_dict['root']) class GitSpec(object): def __init__(self, url, branch, subdir): self.url = url self.branch = branch self.subdir = subdir