From 44cb708351309d2c456eb60a726cc8a49627798f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 27 Jul 2018 11:58:18 +0300 Subject: Drop: build-dep on pylint3 --- debian/control | 1 - pipelines/debian.ick | 122 ++++++++++++++++++++++++++++++++++++------------- pipelines/systrees.ick | 30 +++++++++++- 3 files changed, 119 insertions(+), 34 deletions(-) diff --git a/debian/control b/debian/control index ae4a4b1..9e40b3a 100644 --- a/debian/control +++ b/debian/control @@ -16,7 +16,6 @@ Build-Depends: debhelper (>= 9~), python3-requests, python-requests, pycodestyle, - pylint3, gunicorn3, python3-yaml, cmdtest, diff --git a/pipelines/debian.ick b/pipelines/debian.ick index c6fe573..e93b2ee 100644 --- a/pipelines/debian.ick +++ b/pipelines/debian.ick @@ -5,6 +5,56 @@ pipelines: + # Install APT signing keys from the apt_signing_pub_keys parameter. + # The parameter should contain a dict, where each item is the name + # of a signing key, and is used as the filename in + # /etc/apt/trusted.gpg.d. The dict value is the actual public key, + # in gpg --export --armore format. + + - pipeline: ick/add_apt_signing_keys + parameter: + - apt_signing_pub_keys + actions: + - python: | + for name, key in params['apt_signing_pub_keys'].items(): + filename = '/etc/apt/trusted.gpg.d/{}.asc'.format(name) + print('Writing key to', filename) + with open(filename, 'w') as f: + f.write(key) + where: container + + # Install APT sources listed in the parameter apt_sources by + # creating /etc/apt/sources.list.d/ick_apt_sources.list from the + # list. Each list item must be a dict with fields url, codename, + # section. For example: + # + # apt_sources: + # - url: deb.example.com + # dist: unstable + # section: main + # key: CAFEBEEF + # - url: deb.example.com + # dist: unstable + # section: contrib + # key: CAFEBEEF + + - pipeline: ick/add_apt_sources + parameter: + - apt_sources + actions: + - python: | + apts = params['apt_sources'] + filename = '/etc/apt/sources.list.d/ick_apt_sources.list' + print('Creating', filename) + with open(filename, 'w') as f: + for apt in apts: + f.write('deb {url} {dist} {section}\n'.format(**apt)) + where: container + + - shell: | + apt-get update + where: container + # Install build-dependencies for a Debian build, by checking # debian/control for what's missing. The sources parameter is used # to find all the source trees: for each location, if debian/oontrol @@ -18,8 +68,9 @@ pipelines: import os, subprocess def RUN(*args, **kwargs): - print('Executing:', args) - print('kwargs:', kwargs) + print('Executing:', args, kwargs) + if 'check' not in kwargs: + kwargs['check'] = True return subprocess.run(args, **kwargs) def OUT(*args, **kwargs): @@ -27,11 +78,11 @@ pipelines: return x.stdout.decode('UTF-8') def ERR(*args, **kwargs): - x = RUN(*args, stderr=subprocess.PIPE, **kwargs) + x = RUN(*args, stderr=subprocess.PIPE, check=False, **kwargs) return x.stderr.decode('UTF-8') def is_package_name(s): - ok = "abcdefghijklmnopqrstyvwxyz" + ok = "abcdefghijklmnopqrstyuvwxyz" ok += ok.upper() ok += "0123456789.-" return all(c in ok for c in s) @@ -41,11 +92,14 @@ pipelines: dirname = source['location'] control = os.path.join(dirname, 'debian', 'control') if os.path.exists(control): + RUN('apt', 'show', 'python-coverage-test-runner', check=False) + RUN('apt', 'show', 'python3-coverage-test-runner', check=False) err = ERR('dpkg-checkbuilddeps', cwd=dirname) print('err', err) pat = 'Unmet build dependencies: ' if pat in err: parts = err.split(pat) + print('parts:', repr(parts)) missing = [x for x in parts[1].split() if is_package_name(x)] print('missing', missing) for name in missing: @@ -69,38 +123,44 @@ pipelines: import os, subprocess def RUN(*args, **kwargs): - print('Executing:', args) - print('kwargs:', kwargs) - return subprocess.run(args, **kwargs) + print('Executing:', args, kwargs) + return subprocess.run(args, check=True, **kwargs) def OUT(*args, **kwargs): - x = RUN(*args, stdout=subprocess.PIPE, **kwargs) - return x.stdout.decode('UTF-8') + x = RUN(*args, stdout=subprocess.PIPE, **kwargs) + return x.stdout.decode('UTF-8') sources = params['sources'] + distribution = params['distribution'] for source in sources: - dirname = source['location'] - control = os.path.join(dirname, 'debian', 'control') - if os.path.exists(control): - src = OUT('dpkg-parsechangelog', '-S', 'Source') - print('src', src) - - ver = OUT('dpkg-parsechangelog', '-S', 'Version') - print('ver', ver) - - if '-' in ver: - parts = ver.split('-') - ver = '-'.join(ver_parts[:-1]) - ver += '.0ci{}-1'.format(os.environ['BUILD_NUMBER']) - print('ver', ver) - - tarball = '../{}_{}.orig.tar.xz'.format(src, ver) - print('tarball', tarball) - - RUN('dch', '-v', ver, 'CI build') - RUN('dch', '-r', '') - RUN('sh', '-c', 'git archive HEAD | xz > {}'.format(tarball)) - RUN('dpkg-buildpackage', '-us', '-uc') + dirname = source['location'] + control = os.path.join(dirname, 'debian', 'control') + print('looking for', control) + if os.path.exists(control): + print('building in', dirname) + src = OUT('dpkg-parsechangelog', '-S', 'Source', cwd=dirname) + src = src.strip() + print('src', repr(src)) + assert src + + ver = OUT('dpkg-parsechangelog', '-S', 'Version', cwd=dirname) + print('ver', repr(ver)) + assert ver + + if '-' in ver: + parts = ver.split('-') + ver = '-'.join(parts[:-1]) + ver += '.0ci{}'.format(os.environ['BUILD_NUMBER']) + print('ver', repr(ver)) + + tarball = '../{}_{}.orig.tar.xz'.format(src, ver) + print('tarball', repr(tarball)) + + RUN('dch', '-v', '{}-1'.format(ver), '--distribution', distribution, + '--force-distribution', 'CI build', cwd=dirname) + RUN('dch', '-r', '', cwd=dirname) + RUN('sh', '-c', 'git archive HEAD | xz > {}'.format(tarball), cwd=dirname) + RUN('dpkg-buildpackage', '-us', '-uc', cwd=dirname) where: container # Upload all *.changes files at the root of the workspace. diff --git a/pipelines/systrees.ick b/pipelines/systrees.ick index d675a48..b6da798 100644 --- a/pipelines/systrees.ick +++ b/pipelines/systrees.ick @@ -37,11 +37,37 @@ pipelines: where: host # Prepare container for building: use the artifact named in the - # artifact_name parameter. + # systree_name parameter. - pipeline: ick/setup_container parameters: - - artifact_name + - systree_name actions: - action: populate_systree + name_from: systree_name where: host + + # Archive current systree as the artifact named in the artifact_name + # parameter. + + - pipeline: ick/archive_systree + parameters: + - artifact_name + actions: + - archive: systree + name_from: artifact_name + where: host + + # In a Debian container, install packages listed in the packages + # parameter. + + - pipeline: ick/install_debian_packages + parameters: + - packages + actions: + - python: | + import os, subprocess + def runcmd(argv, **kwargs): + subprocess.check_call(argv, **kwargs) + runcmd(['apt-get', 'install', '-y'] + params['packages']) + where: container -- cgit v1.2.1