path: root/jenkinstool

#!/usr/bin/python
#
# jenkinstool -- manipulate jobs in a running Jenkins instance
#
# Copyright 2012  Lars Wirzenius
# 
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


import cliapp
import json
import logging
import StringIO
from xml.etree import ElementTree as ET

import simplejenkinsapi


have_vcs_copy = '''
have_vcs_copy=yes
'''

prelude = '''

set -eu

if %(global.pretend)s; then exit 0; fi

url_exists()
{
    wget -q -O /dev/null "$1"
}    

upstream_version()
{
    if [ -e setup.py ]
    then
        python setup.py --version && return
    fi

    if [ -e debian/changelog ]
    then
        dpkg-parsechangelog | sed -n '/^Version: /s///p' | sed 's/-[^-]*$//'
        return
    fi

    echo "Can't figure out upstream version" 1>&2
    exit 1
}

commit_date()
{
    if [ -e .bzr ]
    then
        commit_timestamp="$(bzr log -l1 --timezone=utc  | 
                            sed -n '/^timestamp: /s///p')"
    elif [ -e .git ]
    then
        commit_timestamp="$(git log -1 --date=iso | sed -n '/^Date:/s///p')"
    else
        echo "UNKNOWN-COMMIT-TIME"
        return
    fi

    date -u --date="$commit_timestamp" +%%Y%%m%%dT%%H%%M%%S
}

project="%(project.name)s"
artifacts="%(global.artifacts)s/${project}"
project_artifacts="$artifacts/$project"
project_url="%(global.artifacts-url)s/$project"

vcsworkspace="/var/lib/jenkins/workspace/${project}_localhost_vcs-update/"
workdir="$(pwd)"
if [ -e "$vcsworkspace" ] || [ "$have_vcs_copy" = yes ]]
then
    if [ -e "$vcsworkspace" ]
    then
        cd "$vcsworkspace"
    fi

    # Split into two so that if getting the upstream version fails, the
    # script fails.
    version="$(upstream_version)"
    version="$version.vcs$(commit_date)"

    debianversion="${version}-1"
    debianurl="%(global.artifacts-url)s/debian"

    tarball="${project}-${version}.tar.gz"
    tarball_url="${project_url}/$tarball"

    if [ -d debian ]
    then
        sourcepkg="$(dpkg-parsechangelog | awk '/^Source:/ { print $2 }')"
        case "$sourcepkg" in
            lib*) subdir=$(echo "$sourcepkg" | cut -c1-4) ;;
            *)    subdir=$(echo "$sourcepkg" | cut -c1) ;;
        esac
        poolurl="%(global.artifacts-url)s/debian/pool/main/$subdir/$sourcepkg"

        origtgz="${sourcepkg}_${version}.orig.tar.gz"
        changes="${sourcepkg}_${debianversion}_source.changes"
        dsc="${sourcepkg}_$debianversion.dsc"
        dsc_url="$poolurl/$dsc"
        archany_changes="${sourcepkg}_${version}-1_%(host.arch)s.changes"
        archall_changes="${sourcepkg}_${version}-1_all.changes"
    fi

    cd "$workdir"
else
    echo "$vcsworkspace does not exist, not setting version, etc"
fi

'''


class JobGenerator(object):

    '''Generate Jenkins jobs from a JSON specification file.'''
    
    def __init__(self, pretend, pbuilder_max_age, artifacts_url, jenkins_host):
        self.pretend = pretend
        self.pbuilder_max_age = pbuilder_max_age
        self.artifacts_url = artifacts_url
        self.jenkins_host = jenkins_host

    def generate(self, config):
        '''Generate all the jobs for a given config.
        
        Yield (job_id, config_xml) tuples.
        
        '''

        # Create a job to trigger all other jobs.
        a1trigger = self.trigger_everything_job(config)
        jobs = [a1trigger]
        
        # Setup reprepro on Jenkins host.
        reprepro = self.reprepro_setup_job(config)
        jobs[-1].add_build_trigger(reprepro.job_id())
        jobs.append(reprepro)

        # Create or update pbuilder tarballs.
        for host in config['hosts']:
            jc = self.pbuilder_create_job(host)
            jobs[-1].add_build_trigger(jc.job_id())
            jobs.append(jc)

        # Trigger all project pipelines.
        trigger = self.trigger_projects_job(config)
        jobs[-1].add_build_trigger(trigger.job_id())
        jobs.append(trigger)
        
        # Create a pipeline for each project. Every job in the pipeline
        # triggers the next job. The first job is not triggered yet,
        # we set up that later.
        pipelines = {}
        for project in config['projects']:
            prev = None
            for jc in self.generate_project(config, project):
                if prev:
                    prev.add_build_trigger(jc.job_id())
                    pipelines[project['name']].append(jc)
                else:
                    pipelines[project['name']] = [jc]
                prev = jc
                jobs.append(jc)

        # Add triggers for each project pipeline: the last job of each of
        # its build dependencies triggers the first job in the project's
        # pipeline. Projects without build dependencies are triggered by
        # the trigger-everything job.
        for project in config['projects']:
            first = pipelines[project['name']][0]
            deps = project.get('build-depends', [])
            if deps:
                for dep in deps:
                    pipeline = pipelines[dep]
                    pipeline[-1].add_build_trigger(first.job_id())
            else:
                trigger.add_build_trigger(first.job_id())
                
        return [(jc.job_id(), jc.tostring()) for jc in jobs]

    def trigger_everything_job(self, config):
        '''Create job that triggers every other job.'''

        host = config['hosts'][0]
        project = {'name': 'a1_trigger'}
        jc = self.create_job_config(host, project, 'trigger')
        jc.set_description('A1 Trigger All Jobs')
        return jc

    def trigger_projects_job(self, config):
        '''Create job that triggers pipelines for projects.'''

        host = config['hosts'][0]
        project = {'name': 'a2_trigger_projects'}
        jc = self.create_job_config(host, project, 'trigger')
        jc.set_description('A2 Project Trigger')
        return jc

    def reprepro_setup_job(self, config):
        '''Create job that sets up reprepro.'''

        host = { 'name': 'localhost' }
        project = { 'name': 'reprepro-setup' }
        jc = self.create_job_config(host, project, 'reprepro-setup')
        script = '''
if %(global.pretend)s; then exit 0; fi
base="%(global.repreprobase)s"
mkdir -p "$base"
mkdir -p "$base/incoming"
mkdir -p "$base/incoming.temp"
mkdir -p "$base/conf"

cat <<end > "$base/conf/uploaders"
allow * by anybody
end
'''

        archs = set()
        dists = set()
        for host in config['hosts']:
            if 'arch' in host:
                archs.add(host['arch'])
            if 'debian-dist' in host:
                dists.add(host['debian-dist'])

        script += '''
rm -f "$base/conf/distributions"
'''
        for dist in dists:
            script += '''
cat <<end >> "$base/conf/distributions"
Codename: %(dist)s
Suite: %(dist)s
Origin: code.liw.fi
Description: packages from code.liw.fi built for Debian squeeze
Architectures: source %(archs)s
Components: main
Uploaders: uploaders
Tracking: keep includechanges

end
''' % {
    'dist': dist,
    'archs': ' '.join(archs),
}

        script += '''
cat <<end > "$base/conf/incoming"
Name: default
IncomingDir: incoming
TempDir: incoming.temp
Allow: %(dists)s
end
''' % {
    'dists': ' '.join(dists),
}

        script += '''
reprepro -b "$base" export
'''

        jc.add_shell_command(script)
        return jc

    def pbuilder_create_job(self, host):
        '''Create job create/update pbuilder base.tgz.'''

        project = { 'name': 'pbuilder-create' }
        jc = self.create_job_config(host, project, 'pbuilder-create')
        jc.set_description(
            'Create or update pbuilder base.tgz on %(host.ssh-target)s')

        jc.add_ssh_command('''
basetgz=/var/cache/pbuilder/base.tgz
if [ -e "$basetgz" ]
then
    if find $(dirname "$basetgz") -maxdepth 1 \
        -name $(basename "$basetgz") -mtime +%(global.pbuilder-max-age)s |
        grep .
    then
        opts="--update --override-config"
    else
        echo "$basetgz is pretty new, keeping it"
        exit
    fi
else
    opts="--create"
fi
if %(global.pretend)s
then
    echo "PRETEND: pbuilder create/update $basetgz"
else
    sudo pbuilder $opts \
        --distribution "%(host.debian-dist)s" \
        --mirror "%(global.debian-mirror)s"  \
        --othermirror \
            "deb %(global.artifacts-url)s/debian %(host.debian-dist)s main"
fi
''')

        return jc
        
    def generate_project(self, config, project):
        '''Generate all the jobs for a given project.'''

        localhost = {
            'name': 'localhost',
            'arch': 'irrelevant',
        }
        main_host = config['hosts'][0]
        jobs = []
        
        jobs.append(self.vcs_watch_job(localhost, project))

        for host in config['hosts']:
            jobs.append(self.prepare_worker_job(host, project))
            jobs.append(self.build_job(host, project))

        jobs.append(self.tarball_job(main_host, project))

        done = set()
        for host in config['hosts']:
            if host['debian-dist'] not in done:
                jobs.append(self.dsc_job(host, project))
                jobs.append(self.process_incoming_job(
                    config, localhost, project, 'dsc_%s' % host['name']))
                done.add(host['debian-dist'])

        build_all_dists = set()
        for host in config['hosts']:
            build_all = host['debian-dist'] not in build_all_dists
            build_all_dists.add(host['debian-dist'])
            jobs.append(self.deb_job(host, project, build_all))
            jobs.append(self.process_incoming_job(
                config, localhost, project, 'deb_%s' % host['name']))

        return jobs

    def vcs_watch_job(self, host, project):
        '''Generate a job watch a VCS repository.'''

        jc = self.create_job_config(host, project, 'vcs-update')
        jc.set_description(
            'Watch a version control repository: %(project.vcs)s')
        if 'bzr' in project:
            jc.set_bzr(project['bzr'])
        elif 'git' in project:
            jc.set_git(project['git'])
        else:
            raise cliapp.AppException('Unknown VCS')
        return jc

    def prepare_worker_job(self, host, project):
        '''Create job to prepare worker for build.
        
        Upgrades build environment, and configures dput.
        
        '''
        
        jc = self.create_job_config(host, project, 'prepare-worker')
        jc.set_description('Upgrade build environment on %(host.ssh-target)s '
            'for %(project.name)s from %(project.vcs)s')

        jc.add_ssh_command('''
cat << EOF | sudo tee /etc/apt/sources.list.d/jenkinstool.list
deb %(global.artifacts-url)s/debian %(host.debian-dist)s main
EOF

sudo apt-get update || true
sudo apt-get --no-remove --allow-unauthenticated -y upgrade

cat << EOF > /var/lib/jenkins/.dput.cf
[jenkins]
fqdn = %(global.dput-host)s
method = scp
incoming = /var/www/jenkins/debian/incoming
allow_unsigned_uploads = 1
check_version = 0
run_dinstall = 0
EOF
''')

        return jc

    def build_job(self, host, project):
        '''Generate a job to build a given project on a given host.'''

        jc = self.create_job_config(host, project, 'build')
        jc.set_description('Build %(project.name)s from %(project.vcs)s '
                            'on %(host.ssh-target)s')

        self.add_clear_workspace(jc)
        self.add_get_source_from_vcs(jc)
        self.add_rsync_workspace_to_target(jc)
        if not self.pretend:
            jc.add_ssh_command(have_vcs_copy + prelude + '''
missing="$(dpkg-checkbuilddeps 2>&1 | 
           sed -n '/^.*: Unmet build dependencies: /s///p' | 
           sed 's/([^)]*)//g' |
           tr -s ' ' '\n')"
case "$missing" in
    ?*) sudo apt-get install --no-remove --allow-unauthenticated -y $missing ;;
esac
''')
            jc.add_ssh_command('\n'.join(project['build-commands']))
        self.add_rsync_workspace_from_target(jc)

        return jc

    def tarball_job(self, host, project):
        '''Create a job to create upstream release tarball.'''

        jc = self.create_job_config(host, project, 'tarball')
        jc.set_description('Create release tarball for %(project.name)s '
                            'from %(project.vcs)s')
        self.add_clear_workspace(jc)
        self.add_get_source_from_vcs(jc)
        self.add_rsync_workspace_to_target(jc)
        jc.add_ssh_command(have_vcs_copy + prelude + '''
if ! url_exists "$tarball_url"
then
    tempdir="$(mktemp -d)"
    dirname="$tempdir/${project}-${version}"
    if [ -e .bzr ]
    then
        bzr export "$dirname"
    elif [ -e .git ]
    then
        git archive --prefix="${project}-${version}/" HEAD |
            tar -C "$tempdir" -xf -
    else
        echo "Unknown VCS" 1>&2
        exit 1
    fi

    find . -delete
    tar -C "$tempdir" -caf "$tarball" --anchored \
        --exclude "${project}-${version}/debian" \
        "${project}-${version}"
else
    echo "$artifacts/$tarball already exists, ignoring rebuilt version"
fi
''')
        self.add_rsync_workspace_from_target(jc)

        jc.add_shell_command(have_vcs_copy + prelude + '''
mkdir -p "$artifacts"
for x in *
do
    if [ ! -e "$artifacts/$x" ]
    then
        mv "$x" "$artifacts/."
    fi
done
''')

        return jc

    def dsc_job(self, host, project):
        '''Create a job to create Debian source package.'''

        jc = self.create_job_config(host, project, 'dsc')
        jc.set_description('Create Debian source package for %(project.name)s '
                            'from %(project.vcs)s on %(host.ssh-target)s')

        self.add_clear_workspace(jc)
        self.add_get_source_from_vcs(jc)
        self.add_rsync_workspace_to_target(jc)
        jc.add_ssh_command(have_vcs_copy + prelude + '''\
if url_exists "$dsc_url"
then
    echo "$dsc_url already exists, not rebuilding"
    exit 0
else
    wget -O "../$origtgz" "$project_url/$tarball"
    dch -v "$debianversion" --distribution "%(host.debian-dist)s" "CI build"
    dch -r ''
    debuild -S -us -uc -i
    dput -f jenkins "../$changes"
fi
''')
        self.add_rsync_workspace_from_target(jc)
        
        return jc

    def process_incoming_job(self, config, localhost, project, suffix):
        '''Create job that processing reprepro's incoming queue.'''

        jc = self.create_job_config(localhost, project, 'incoming_' + suffix)
        jc.set_description('Process reprepro incoming queue')
        jc.add_shell_command(
            'reprepro -b "%(global.repreprobase)s" processincoming default')
        return jc

    def deb_job(self, host, project, build_all):
        '''Create a job to build a Debian binary package on a given host.'''

        jc = self.create_job_config(host, project, 'deb')
        jc.set_description('Build Debian binary package for %(project.name)s '
                            'from %(project.vcs)s on %(host.ssh-target)s')

        self.add_clear_workspace(jc)
        self.add_get_source_from_vcs(jc)

        self.add_rsync_workspace_to_target(jc)
        
        script = have_vcs_copy + prelude
        if build_all:
            script += '''
if grep '^Architecture:.* any' debian/control > /dev/null
then
    url="$poolurl/$archany_changes"
else
    url="$poolurl/$archall_changes"
fi
if url_exists "$url"
then
    echo "$poolurl/$archall_changes exists, not rebuilding"
    exit 0
fi
binopt=-b
'''
        else:
            script += '''
if ! grep '^Architecture:.* any' debian/control > /dev/null
then
    echo No architecture specific packages to build, skipping.
    exit 0
fi
if url_exists "$poolurl/$archany_changes"
then
    echo "$poolurl/$archany_changes exists, not rebuilding"
    exit 0
fi
binopt=-B
'''

        jc.add_ssh_command(script + '''
curdir="$(pwd)"
temp="$(mktemp -d)"
cd "$temp"
dget -u "$dsc_url"
mkdir result
if man pbuilder | grep -e --allow-untrusted
then
    untrusted="--allow-untrusted"
else
    untrusted=""
fi
sudo pbuilder --update
sudo pbuilder --build --buildresult result $untrusted \
    --debbuildopts $binopt *.dsc
dput jenkins "result/${sourcepkg}_${version}-1_%(host.arch)s.changes"
cd "$curdir"
rm -rf "$temp"
''')
        self.add_rsync_workspace_from_target(jc)

        return jc

    def create_job_config(self, host, project, suffix):
        jc = simplejenkinsapi.JobConfig()
        jc.set_job_suffix(suffix)
        if 'bzr' in project:
            project['vcs'] = project['bzr']
        elif 'git' in project:
            project['vcs'] = project['git']
        jc.add_param_dict('project', project)
        jc.add_param_dict('host', host)
        jc.add_param_dict('global', {
            'artifacts': '/var/www/jenkins',
            'artifacts-url': self.artifacts_url,
            'dput-host': self.jenkins_host,
            'repreprobase': '/var/www/jenkins/debian',
            'debian-mirror': 'http://cdn.debian.net/debian',
            'pretend': 'true' if self.pretend else 'false',
            'pbuilder-max-age': str(self.pbuilder_max_age),
        })
        return jc

    def add_rsync_workspace_to_target(self, jc):
        jc.add_ssh_command('mkdir -p "%(host.directory)s"')
        jc.add_shell_command(
            'rsync -aHS --delete-before . '
            '"%(host.ssh-target)s:%(host.directory)s/."')

    def add_rsync_workspace_from_target(self, jc):
        jc.add_shell_command(
            'rsync -aHS --delete-before '
                '"%(host.ssh-target)s:%(host.directory)s/." .')

    def add_clear_workspace(self, jc):
        jc.add_shell_command('find . -delete')

    def add_get_source_from_vcs(self, jc):
        jc.add_shell_command(prelude + 'cp -a "$vcsworkspace/." .')

class JenkinsTool(cliapp.Application):

    '''Manipulate jobs in running Jenkins CI server.'''
    
    def add_settings(self):
        self.settings.string(
            ['url', 'jenkins-url'], 
            'URL to Jenkins server; this is a required setting')
        self.settings.boolean(
            ['pretend-jobs'],
            'make generate-jobs create jobs that merely pretend to do things')
        self.settings.integer(
            ['pbuilder-max-age'],
            'update pbuilder base.tgz if older than DAYS',
            metavar='DAYS',
            default=7)
        self.settings.string(
            ['artifacts-url'],
            'URL to artifact repository')
        self.settings.string(
            ['jenkins-host'],
            'address of host running jenkins')

    def cmd_list_jobs(self, args):
        '''List all jobs on the server.'''
        jenkins = simplejenkinsapi.Jenkins(self.settings['url'])
        for job_id in jenkins.list_jobs():
            self.output.write('%s\n' % job_id)

    def cmd_delete_job(self, args):
        '''Remove specified jobs from server.'''
        jenkins = simplejenkinsapi.Jenkins(self.settings['url'])
        for job_id in args:
            jenkins.delete_job(job_id)

    def cmd_delete_all_jobs(self, args):
        '''Remove all jobs from server.'''
        jenkins = simplejenkinsapi.Jenkins(self.settings['url'])
        for job_id in jenkins.list_jobs():
            jenkins.delete_job(job_id)

    def cmd_create_job(self, args):
        '''Create a job on the server.'''
        if len(args) != 2:
            raise cliapp.AppException(
                'Need JOB-ID and CONFIG-FILENAME arguments')
        job_id, config_xml_filename = args
        with open(config_xml_filename) as f:
            jenkins = simplejenkinsapi.Jenkins(self.settings['url'])
            jenkins.create_job(job_id, f.read())

    def cmd_update_job(self, args):
        '''Update a job's config on the server.'''
        if len(args) != 2:
            raise cliapp.AppException(
                'Need JOB-ID and CONFIG-FILENAME arguments')
        job_id, config_xml_filename = args
        with open(config_xml_filename) as f:
            jenkins = simplejenkinsapi.Jenkins(self.settings['url'])
            jenkins.update_job(job_id, f.read())

    def cmd_generate_jobs(self, args):
        '''Generate jobs based on JSON format job specification file.'''

        self.settings.require('artifacts-url')

        jenkins = simplejenkinsapi.Jenkins(self.settings['url'])
        job_generator = JobGenerator(self.settings['pretend-jobs'],
                                     self.settings['pbuilder-max-age'],
                                     self.settings['artifacts-url'],
                                     self.settings['jenkins-host'])
        created_job_ids = set()
        for filename in args:
            with open(filename) as f:
                try:
                    config = json.load(f)
                except ValueError, e:
                    raise cliapp.AppException('%s: %s' % (filename, str(e)))
            for job_id, config_xml in job_generator.generate(config):
                logging.debug('job_id = %s' % job_id)
                logging.debug('config_xml:\n%s' % config_xml)
                if job_id in jenkins.list_jobs():
                    jenkins.update_job(job_id, config_xml)
                else:
                    jenkins.create_job(job_id, config_xml)
                created_job_ids.add(job_id)
        for job_id in jenkins.list_jobs():
            if job_id not in created_job_ids:
                jenkins.delete_job(job_id)
                

JenkinsTool(version=simplejenkinsapi.__version__).run()