path: root/icklib/project.py

# Copyright 2015  Lars Wirzenius
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# =*= License: GPL-3+ =*=


import os
import StringIO
import subprocess
import time

import cliapp

import icklib


class Project(object):

    def __init__(self, name):
        self.name = name
        self.progress = None
        self.git_specs = []
        self.repo_signing_key = None
        self.package_signing_key = None
        self.commands = []
        self.local_commands = []
        self.env = {}
        self.dput_targets = []
        self.dput_unstable = []

        self.pipelines = {}

    def set_progress(self, progress):
        self.progress = progress

    def add_git_spec(self, git_spec):
        self.git_specs.append(git_spec)

    def set_repo_signing_key(self, key_id):
        self.repo_signing_key = key_id

    def set_package_signing_key(self, key_id):
        self.package_signing_key = key_id

    def add_shell_command(self, command):
        self.commands.append(command)

    def add_local_command(self, command):
        self.local_commands.append(command)

    def add_dput_target(self, dput_target):
        self.dput_targets.append(dput_target)

    def add_dput_unstable(self, dput_target):
        self.dput_unstable.append(dput_target)

    def set_build_pipelines(self, pipelines):
        self.pipelines = pipelines

    def set_env(self, env):
        self.env = env

    def build(self, statedir, targets):
        for pipeline_name, pipeline_steps in self.pipelines.items():
            pipeline = icklib.BuildPipeline(pipeline_name)
            pipeline.set_build_steps(pipeline_steps)
            run_state = icklib.RunState()
            run_state.pipeline = pipeline
            run_state.progress = self.progress
            run_state.logger = icklib.Logger()
            run_state.log_catcher = StringIO.StringIO()
            run_state.logger.add_output_file(run_state.log_catcher)
            self.run_state = run_state

            self.progress['pipeline'] = pipeline_name
            run_state.logger.important(
                'Build log starts {timestamp}',
                timestamp=self.current_timestamp())
            run_state.logger.important(
                'Building project {project}',
                project=self.name)
            run_state.logger.important(
                'Executing pipeline {name}', name=pipeline_name)

            run_state.started = time.time()
            try:
                pipeline.build(self, statedir, targets, run_state)
            except:
                if run_state.build_info is not None:
                    run_state.build_info.status = 'FAILURE'
                    run_state.build_info.duration = (
                        time.time() - run_state.started)
                    run_state.build_info.save()
                run_state.logger.important('')
                run_state.logger.important('Build status: FAILURE')
                run_state.logger.important(
                    'Build log ends {timestamp}',
                    timestamp=self.current_timestamp())
                raise

            if run_state.build_info is not None:
                run_state.logger.important(
                    'Build status: {status}',
                    status=run_state.build_info.status)
            run_state.logger.important(
                'Build log ends {timestamp}',
                timestamp=self.current_timestamp())

    def current_timestamp(self):
        timezone = '{sign}{h}{m}'.format(
            sign='-' if time.timezone < 0 else '+',
            h=abs(time.timezone) / 3600,
            m=(abs(time.timezone) % 3600) / 60)
        return '{timestamp} {timezone}'.format(
            timestamp=time.strftime('%Y-%m-%d %H:%M:%S'),
            timezone=timezone)

    def run_locally(self, argv, displayed_cmd, cwd):
        self.report_running_locally(displayed_cmd)
        self.run_state.logger.important(
            'On local shell run {command}',
            command=displayed_cmd)

        whole_argv = self.create_argv_with_env() + argv

        line_logger = icklib.LineLogger(self.progress, self.run_state.logger)
        with self.run_state.logger:
            self.run_state.logger.chatty('+ {argv}', argv=' '.join(whole_argv))
            cliapp.runcmd(
                whole_argv,
                stderr=subprocess.STDOUT,
                stdout_callback=line_logger.log_lines,
                cwd=cwd)
            line_logger.log_rest()

        self.report_finished_on_target()

    def create_argv_with_env(self):
        env_argv = ['env']
        for name, value in self.env.items():
            env_argv.append('%s=%s' % (name, value))
        return env_argv

    def report_running_locally(self, displayed_cmd):
        progress = {
            'target': 'local',
            'command': '\\n'.join(displayed_cmd.split('\n')),
        }
        for i in range(self.run_state.progress.max_output_lines):
            progress['line%d' % i] = ''
        for key, value in progress.items():
            self.run_state.progress[key] = value

    def run_argv_on_target(self, target, argv, displayed_cmd, remote_cwd):
        self.report_running_on_target(target, displayed_cmd)
        self.run_state.logger.important(
            'On {address} run {command}',
            address=target.address, command=displayed_cmd)

        env_argv = ['env']
        for name, value in self.env.items():
            env_argv.append('%s=%s' % (name, value))
        whole_argv = self.create_argv_with_env() + argv

        line_logger = icklib.LineLogger(self.progress, self.run_state.logger)
        with self.run_state.logger:
            self.run_state.logger.chatty('+ %s' % ' '.join(whole_argv))
            target.ssh_runcmd(
                whole_argv,
                remote_cwd=remote_cwd,
                tty=False,
                stderr=subprocess.STDOUT,
                stdout_callback=line_logger.log_lines)
            line_logger.log_rest()

        self.report_finished_on_target()

    def report_running_on_target(self, target, displayed_cmd):
        progress = {
            'target': target.name,
            'command': '\\n'.join(displayed_cmd.split('\n')),
        }
        for i in range(self.run_state.progress.max_output_lines):
            progress['line%d' % i] = ''
        for key, value in progress.items():
            self.run_state.progress[key] = value

    def report_finished_on_target(self):
        keys = ['target', 'command']
        for i in range(self.run_state.progress.max_output_lines):
            keys.append('line%d' % i)
        for key in keys:
            self.run_state.progress[key] = ''

    def copy_cleanly_to_target(self, target, remote_tmp):
        self.run_state.logger.chatty(
            'Copy helper script (./cleanly) to target')
        local_cleanly_path = os.path.join(
            os.path.dirname(icklib.__file__), 'cleanly')
        remote_cleanly_path = os.path.join(remote_tmp, 'cleanly')
        cliapp.runcmd(
            ['rsync', '-s', local_cleanly_path,
             '%s:%s' % (target.address, remote_cleanly_path)])
        return remote_cleanly_path

    def run_cleanly(self, target, remote_git_dir, cleanly_path, args):
        self.run_argv_on_target(
            target,
            ['python', cleanly_path] + args,
            'cleanly ' + args[-1],
            remote_git_dir)


def create_projects_from_ick(ick, wanted_names):
    pipelines = {
        'shell': [
            icklib.LoadProjectInfo,
            icklib.CloneGits,
            icklib.FindCurrentGitCommits,
            icklib.FindNewCommitToBuildForShell,
            icklib.CreateBuildInfo,
            icklib.RunShellCommandsOnEachTarget,
            icklib.FinishBuildInfo,
            icklib.SaveProjectInfo,
        ],
        'local-shell': [
            icklib.LoadProjectInfo,
            icklib.CloneGits,
            icklib.FindCurrentGitCommits,
            icklib.FindNewCommitToBuildForLocalShell,
            icklib.CreateBuildInfo,
            icklib.RunLocalShellCommands,
            icklib.FinishBuildInfo,
            icklib.SaveProjectInfo,
        ],
        'debian-ci': [
            icklib.LoadProjectInfo,
            icklib.CloneGits,
            icklib.FindCurrentGitCommits,
            icklib.FindNewCommitToBuildForDebianCI,
            icklib.CreateBuildInfo,
            icklib.CollectDebianInfoAboutTargets,
            icklib.CreateCITarball,
            icklib.CreateDebianSourcePackagesForCI,
            icklib.CreateCIDebianBinaryPackages,
            icklib.FindDebianChangesFiles,
            icklib.SetupAPTRepository,
            icklib.UploadDebianPackagesToCIRepo,
            icklib.FinishBuildInfo,
            icklib.SaveProjectInfo,
        ],
        'debian-release': [
            icklib.LoadProjectInfo,
            icklib.CloneGits,
            icklib.FindCurrentGitCommits,
            icklib.FindTagsToBuildForDebianRelease,
            icklib.CreateBuildInfo,
            icklib.CollectDebianInfoAboutTargets,
            icklib.CreateReleaseTarballs,
            icklib.CreateDebianSourcePackagesForRelease,
            icklib.CreateDebianBinaryPackagesForRelease,
            icklib.FindDebianChangesFiles,
            icklib.SetupAPTRepository,
            icklib.UploadDebianPackagesToCIRepo,
            icklib.PublishDebianPackages,
            icklib.FinishBuildInfo,
            icklib.SaveProjectInfo,
        ],
    }

    projects = []
    if not wanted_names:
        wanted_names = ick.get('projects', {}).keys()

    for project_name, project_dict in ick.get('projects', {}).items():
        if project_name in wanted_names:
            p = Project(project_name)
            for git_spec in create_git_specs_for_project(project_dict):
                p.add_git_spec(git_spec)
            if 'repo-signing-key' in ick:
                p.set_repo_signing_key(ick['repo-signing-key'])
            if 'shell' in project_dict:
                p.add_shell_command(project_dict['shell'])
            if 'local-shell' in project_dict:
                p.add_local_command(project_dict['local-shell'])
            if 'debian-release' in project_dict:
                release = project_dict['debian-release']
                dput_targets = release.get('dput-targets', [])
                for dput_target in dput_targets:
                    p.add_dput_target(dput_target)
                dput_unstable = release.get('dput-unstable', [])
                for dput_target in dput_unstable:
                    p.add_dput_unstable(dput_target)
            p.set_build_pipelines(
                dict((x, pipelines[x]) for x in project_dict['pipelines']))
            p.set_env(project_dict.get('env', {}))

            projects.append(p)

    return projects


def create_git_specs_for_project(project_dict):
    if 'git' in project_dict:
        yield GitSpec(project_dict['git'], project_dict['branch'], '.')
    elif 'gits' in project_dict:
        for git_dict in project_dict['gits']:
            yield GitSpec(
                git_dict['git'], git_dict['branch'], git_dict['root'])


class GitSpec(object):

    def __init__(self, url, branch, subdir):
        self.url = url
        self.branch = branch
        self.subdir = subdir