diff options
author | Lars Wirzenius <liw@liw.fi> | 2018-07-27 17:14:41 +0300 |
---|---|---|
committer | Lars Wirzenius <liw@liw.fi> | 2018-07-27 17:14:41 +0300 |
commit | 37147aa3ce743132cb973d8adec54fe7d8b2d08c (patch) | |
tree | d5354f3385e32ad35ab05c742b0d979e4c1ea0fb | |
parent | d8ceea1cb92cbbe5b200e9452a8b9e9645b18957 (diff) | |
download | liw-ci-37147aa3ce743132cb973d8adec54fe7d8b2d08c.tar.gz |
Change: split up ci-prod.ick into smaller files, re-create all
-rw-r--r-- | ci-prod-distixes.ick | 176 | ||||
-rw-r--r-- | ci-prod-programs.ick | 443 | ||||
-rw-r--r-- | ci-prod-systrees.ick | 74 | ||||
-rw-r--r-- | ci-prod-websites.ick | 186 | ||||
-rw-r--r-- | ci-prod.ick | 1099 | ||||
-rw-r--r-- | ickdev.ick | 266 | ||||
-rw-r--r-- | testick.ick | 208 |
7 files changed, 879 insertions, 1573 deletions
diff --git a/ci-prod-distixes.ick b/ci-prod-distixes.ick new file mode 100644 index 0000000..10c8541 --- /dev/null +++ b/ci-prod-distixes.ick @@ -0,0 +1,176 @@ +# Some projetcts to build static web sites from distix repos. + +projects: + + - project: stretch-distix-systree + parameters: + systree_name: systrees-debian-stretch-base + artifact_name: systrees-debian-stretch-distix + packages: + - ikiwiki + - libhighlight-perl + - libimage-magick-perl + - graphviz + - ql-ikiwiki-publish + - locales-all + - distix + - pandoc + - locales-all + notify: [liw@liw.fi] + apt_sources: + - url: http://ci-prod-controller.vm.liw.fi/debian + dist: stretch-ci + section: main + apt_signing_pub_keys: + code.liw.fi: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFrLO7kBEADdz6mHstYmKU5Dp6OSjxWtWaqTDOX1sJdmmaIK/9EKVIH0Maxp + 5kvVO5G6mULLAjv/kLG0MxasHPrq8I2A/y8AqKAGVL8QelwLjQMIFZ30/VbGQPHS + +T5TZXEnoQtNce1GUhFwJ38ZyjjwHBFV9tSec7rZ2Q3YeM3nNnGPf6DacXGfEOPO + HIN4sXAN2hzNXNjKRzTIvxQseb6nr7afUh/SlZ3yhQOCrIzmYlD7tP9WJe7ofL0p + JY4pDQYw8rT6nC2BE/ioemh84kERCT1vCe+OVFlSRuMlqfEv+ZpKQ+itOmPDQ/lM + jpUm1K2hrW/lWpxT/ZxHKo/w1K36J5WshgMZxfUu5BMCL9LMqMcrXNhNjDMfxDMM + 3yBPOvQ4ls6fecOZ/bsFo1p8VzMk/w/eG8vPs5yuNa5XxN95yFMXoOHGb5Xbu8D4 + 6yiW+Af70LbiSNpGdmNdneiGB2fY38NxBukPw5u3S5qG8HedSmMr1RvSr5kHoAAe + UbOY+BYaaKsTAT7+1skUW1o3FJSqoRKCHAzTsMWC6zzhR8hRn7jVrrguH1hGbqq5 + TZSCFQZExuTJ7uXrTLG0WoBXIjB5wWNcSeXn8myUWYB51nJNF4tJBouZOz9JwWGl + kiAQkrHnBttLQWdW9FyjbIoTZMtpvVx+m6ObGTGdGL1cNlLAvWprMXGc+QARAQAB + tDJJY2sgQVBUIHJlcG9zaXRvcnkgc2lnbmluZyBrZXkgKDIwMTgpIDxsaXdAbGl3 + LmZpPokCTgQTAQgAOBYhBKL1uyDoXyxUH3O717Wr+TZVS6PGBQJayzu5AhsDBQsJ + CAcCBhUICQoLAgQWAgMBAh4BAheAAAoJELWr+TZVS6PGB5QQANTcikhRUHwt9N4h + dGc/Hp6CbqdshMoWlwpFskttoVDxQG5OAobuZl5XyzGcmja1lT85RGkZFfbca0IZ + LnXOLLSAu51QBkXNaj4OhjK/0uQ+ITrvL6RQSXNgHiUTR/W2XD1GIUq6nBqe2GSN + 31S1baYKKVj5QIMsi7Dq8ls3BBXuPCE+xTSaNmGWjes2t9pPidcRvxsksCLY1qgw + P1GFXBeMkBQ29kBP87SUL15SIk7OiQLlEURCy5iRls5rt/YEsdEpRWIb0Tm5Nrjv + 2M3VM+iBhfNXTwj0rJ34mlycF1qQmA7YcTEobT7z587GPY0VWzBpQUnEQj7rQWPM + cDYY0b+I6kQ8VKOaL4wVAtE98d7HzFIrIrwhTKufnrWrVDPYsmLZ+LPC1jiF7JBD + SR6Vftb+SdDR9xoE1yRuXbC6IfoW+5/qQNrdQ2mm9BFw5jOonBqchs18HTTf3441 + 6SWwP9fY3Vi+IZphPPi0Gf85oMStgnv/Wnw6LacEL32ek39Desero/D8iGLZernK + Q2mC9mua5A/bYGVhsNWyURNFkKdbFa+/wW3NfdKYyZnsSfo+jJ2luNewrhAY7Kod + GWXTer9RxzTGA3EXFGvNr+BBOOxSj0SfWTl0Olo7J5dnxof+jLAUS1VHpceHGHps + GSJSdir7NkZidgwoCPA7BTqsb5LN + =dXB0 + -----END PGP PUBLIC KEY BLOCK----- + pipelines: + - ick/setup_container + - ick/add_apt_signing_keys + - ick/add_apt_sources + - ick/install_debian_packages + - ick/archive_systree + + - project: dummy_distix + parameters: &distix_params + rsync_src: html + systree_name: systrees-debian-stretch-distix + notify: [liw@liw.fi] + pipelines: &distix_pipelines + - ick/setup_container + - ick/get_sources + - distix_html + + - project: bugs@liw.fi + parameters: + <<: *distix_params + rsync_target: ickliwfi@pieni.net:/srv/http/bugs.liw.fi + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.liw.fi/bugs-liw-fi-distix + pipelines: *distix_pipelines + + - project: tickets@distix.eu + parameters: + <<: *distix_params + rsync_target: ickliwfi@pieni.net:/srv/http/tickets.distix.eu + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.liw.fi/distix-bugs + pipelines: *distix_pipelines + + - project: ick-support + parameters: + <<: *distix_params + rsync_target: ickliwfi@pieni.net:/srv/http/ick-support.liw.fi + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.liw.fi/ick-devel-distix + pipelines: *distix_pipelines + + - project: qvarn-devel + parameters: + <<: *distix_params + rsync_target: static@static.qvarnlabs.net:/srv/http/qvarn-devel.qvarnlabs.net + notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.qvarnlabs.net/qvarn-devel.distix + pipelines: *distix_pipelines + + - project: issues.qvarnlabs.net + parameters: + <<: *distix_params + rsync_target: static@static.qvarnlabs.net:/srv/http/issues.qvarnlabs.net + notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.qvarnlabs.net/issues.distix + pipelines: *distix_pipelines + + - project: qvarn-support + parameters: + <<: *distix_params + notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] + rsync_target: static@static.qvarnlabs.net:/srv/http/qvarn-support.qvarnlabs.net + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.qvarnlabs.net/qvarn-support.distix + pipelines: *distix_pipelines + + - project: defour-support + parameters: + <<: *distix_params + notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] + rsync_target: static@static.qvarnlabs.net:/srv/http/defour-support.qvarnlabs.net + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.qvarnlabs.net/defour-support.distix + pipelines: *distix_pipelines + +pipelines: + + - pipeline: distix_html + parameters: + - sources + - rsync_src + - rsync_target + actions: + - python: | + import os, re, subprocess + def RUN(*args, cwd=None): + print('Executing:', args, 'cwd:', cwd) + subprocess.check_call(args, cwd=cwd) + sources = params['sources'] + + RUN('rm', '-rf', 'html') + for source in sources: + dirname = source['location'] + RUN('distix', 'html', '/workspace/html', cwd=dirname) + where: container + + - action: rsync + where: host + diff --git a/ci-prod-programs.ick b/ci-prod-programs.ick new file mode 100644 index 0000000..68db26d --- /dev/null +++ b/ci-prod-programs.ick @@ -0,0 +1,443 @@ +projects: + +# dummy-stretch is a project that I don't expect to ever build, and it +# doesn't actually work. It's reason to exist is to have a convenient +# place to define comon parameters and a list of pipelines, both of +# which can be referred to in actual projects using YAML magic. + +- project: dummy-stretch + parameters: &dummy_stretch_params + apt_sources: + - url: http://ci-prod-controller.vm.liw.fi/debian + dist: stretch-ci + section: main + systree_name: systrees-debian-stretch-build-essential + distribution: stretch-ci + notify: [liw@liw.fi] + apt_signing_pub_keys: + code.liw.fi: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFrLO7kBEADdz6mHstYmKU5Dp6OSjxWtWaqTDOX1sJdmmaIK/9EKVIH0Maxp + 5kvVO5G6mULLAjv/kLG0MxasHPrq8I2A/y8AqKAGVL8QelwLjQMIFZ30/VbGQPHS + +T5TZXEnoQtNce1GUhFwJ38ZyjjwHBFV9tSec7rZ2Q3YeM3nNnGPf6DacXGfEOPO + HIN4sXAN2hzNXNjKRzTIvxQseb6nr7afUh/SlZ3yhQOCrIzmYlD7tP9WJe7ofL0p + JY4pDQYw8rT6nC2BE/ioemh84kERCT1vCe+OVFlSRuMlqfEv+ZpKQ+itOmPDQ/lM + jpUm1K2hrW/lWpxT/ZxHKo/w1K36J5WshgMZxfUu5BMCL9LMqMcrXNhNjDMfxDMM + 3yBPOvQ4ls6fecOZ/bsFo1p8VzMk/w/eG8vPs5yuNa5XxN95yFMXoOHGb5Xbu8D4 + 6yiW+Af70LbiSNpGdmNdneiGB2fY38NxBukPw5u3S5qG8HedSmMr1RvSr5kHoAAe + UbOY+BYaaKsTAT7+1skUW1o3FJSqoRKCHAzTsMWC6zzhR8hRn7jVrrguH1hGbqq5 + TZSCFQZExuTJ7uXrTLG0WoBXIjB5wWNcSeXn8myUWYB51nJNF4tJBouZOz9JwWGl + kiAQkrHnBttLQWdW9FyjbIoTZMtpvVx+m6ObGTGdGL1cNlLAvWprMXGc+QARAQAB + tDJJY2sgQVBUIHJlcG9zaXRvcnkgc2lnbmluZyBrZXkgKDIwMTgpIDxsaXdAbGl3 + LmZpPokCTgQTAQgAOBYhBKL1uyDoXyxUH3O717Wr+TZVS6PGBQJayzu5AhsDBQsJ + CAcCBhUICQoLAgQWAgMBAh4BAheAAAoJELWr+TZVS6PGB5QQANTcikhRUHwt9N4h + dGc/Hp6CbqdshMoWlwpFskttoVDxQG5OAobuZl5XyzGcmja1lT85RGkZFfbca0IZ + LnXOLLSAu51QBkXNaj4OhjK/0uQ+ITrvL6RQSXNgHiUTR/W2XD1GIUq6nBqe2GSN + 31S1baYKKVj5QIMsi7Dq8ls3BBXuPCE+xTSaNmGWjes2t9pPidcRvxsksCLY1qgw + P1GFXBeMkBQ29kBP87SUL15SIk7OiQLlEURCy5iRls5rt/YEsdEpRWIb0Tm5Nrjv + 2M3VM+iBhfNXTwj0rJ34mlycF1qQmA7YcTEobT7z587GPY0VWzBpQUnEQj7rQWPM + cDYY0b+I6kQ8VKOaL4wVAtE98d7HzFIrIrwhTKufnrWrVDPYsmLZ+LPC1jiF7JBD + SR6Vftb+SdDR9xoE1yRuXbC6IfoW+5/qQNrdQ2mm9BFw5jOonBqchs18HTTf3441 + 6SWwP9fY3Vi+IZphPPi0Gf85oMStgnv/Wnw6LacEL32ek39Desero/D8iGLZernK + Q2mC9mua5A/bYGVhsNWyURNFkKdbFa+/wW3NfdKYyZnsSfo+jJ2luNewrhAY7Kod + GWXTer9RxzTGA3EXFGvNr+BBOOxSj0SfWTl0Olo7J5dnxof+jLAUS1VHpceHGHps + GSJSdir7NkZidgwoCPA7BTqsb5LN + =dXB0 + -----END PGP PUBLIC KEY BLOCK----- + pipelines: &deb_build_pipelines + - ick/setup_container + - ick/add_apt_signing_keys + - ick/add_apt_sources + - ick/get_sources + - ick/install_debian_build_dependencies + - ick/build_deb_ci + - ick/upload_debs + +# dummy-sid is like dummy-stretch, except for building for Debian +# unstable (code name sid). + +- project: dummy-sid + parameters: &dummy_sid_params + <<: *dummy_stretch_params + systree_name: systrees-debian-sid-build-essential + distribution: unstable-ci + pipelines: *deb_build_pipelines + +# Real projects follow. Each project is duplicated with a stretch and +# sid version. + + + +- project: coverage-test-runner-stretch + parameters: + <<: *dummy_stretch_params + sources: &coverage_test_runner_sources + - name: code + repo: git://git.liw.fi/coverage-test-runner + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: coverage-test-runner-sid + parameters: + <<: *dummy_sid_params + sources: *coverage_test_runner_sources + pipelines: *deb_build_pipelines + + + +- project: cliapp-stretch + parameters: + <<: *dummy_stretch_params + sources: &cliapp_sources + - name: code + repo: git://git.liw.fi/cliapp + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: cliapp-sid + parameters: + <<: *dummy_sid_params + sources: *cliapp_sources + pipelines: *deb_build_pipelines + + + +- project: cachedir-stretch + parameters: + <<: *dummy_stretch_params + sources: &cachedir_sources + - name: code + repo: git://git.liw.fi/cachedir + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: cachedir-sid + parameters: + <<: *dummy_stretch_params + sources: *cachedir_sources + pipelines: *deb_build_pipelines + + + +- project: bumper-stretch + parameters: + <<: *dummy_stretch_params + sources: &bumper_sources + - name: code + repo: git://git.liw.fi/bumper + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: bumper-sid + parameters: + <<: *dummy_sid_params + sources: *bumper_sources + pipelines: *deb_build_pipelines + + + +- project: clab-stretch + parameters: + <<: *dummy_stretch_params + sources: &clab_sources + - name: code + repo: git://git.liw.fi/clab + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: clab-sid + parameters: + <<: *dummy_sid_params + sources: *clab_sources + pipelines: *deb_build_pipelines + + + +- project: ttystatus-stretch + parameters: + <<: *dummy_stretch_params + sources: &ttystatus_sources + - name: code + repo: git://git.liw.fi/ttystatus + ref: master + location: src + pipelines: *deb_build_pipelines + +# commented out while pylint3 is uninstallable in sid +# - project: ttystatus-sid +# parameters: +# <<: *dummy_sid_params +# sources: *ttystatus_sources +# pipelines: *deb_build_pipelines + + + +- project: copyright-statement-lint-stretch + parameters: + <<: *dummy_stretch_params + sources: ©right_statement_lint_sources + - name: code + repo: git://git.liw.fi/copyright-statement-lint + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: copyright-statement-lint-sid + parameters: + <<: *dummy_sid_params + sources: *copyright_statement_lint_sources + pipelines: *deb_build_pipelines + + + +- project: py_pgpwordlist-stretch + parameters: + <<: *dummy_stretch_params + sources: &py_pgpwordlist_sources + - name: code + repo: git://git.liw.fi/py_pgpwordlist + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: py_pgpwordlist-sid + parameters: + <<: *dummy_sid_params + sources: *py_pgpwordlist_sources + pipelines: *deb_build_pipelines + + + +- project: distix-stretch + parameters: + <<: *dummy_stretch_params + sources: &distix_sources + - name: code + repo: git://git.liw.fi/distix + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: distix-sid + parameters: + <<: *dummy_sid_params + sources: *distix_sources + pipelines: *deb_build_pipelines + + + +- project: extrautils-stretch + parameters: + <<: *dummy_stretch_params + sources: &extrautils_sources + - name: code + repo: git://git.liw.fi/extrautils + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: extrautils-sid + parameters: + <<: *dummy_sid_params + sources: *extrautils_sources + pipelines: *deb_build_pipelines + + + +- project: cmdtest-stretch + parameters: + <<: *dummy_stretch_params + sources: &cmdtest_sources + - name: code + repo: git://git.liw.fi/cmdtest + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: cmdtest-sid + parameters: + <<: *dummy_sid_params + sources: *cmdtest_sources + pipelines: *deb_build_pipelines + + + +- project: genbackupdata-stretch + parameters: + <<: *dummy_stretch_params + sources: &genbackupdata_sources + - name: code + repo: git://git.liw.fi/genbackupdata + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: genbackupdata-sid + parameters: + <<: *dummy_sid_params + sources: *genbackupdata_sources + pipelines: *deb_build_pipelines + + + +- project: jt-stretch + parameters: + <<: *dummy_stretch_params + sources: &jt_sources + - name: code + repo: git://git.liw.fi/jt + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: jt-sid + parameters: + <<: *dummy_sid_params + sources: *jt_sources + pipelines: *deb_build_pipelines + + + +- project: dupfiles-stretch + parameters: + <<: *dummy_stretch_params + sources: &dupfiles_sources + - name: code + repo: git://git.liw.fi/dupfiles + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: dupfiles-sid + parameters: + <<: *dummy_sid_params + sources: *dupfiles_sources + pipelines: *deb_build_pipelines + + + +- project: liw-automation-stretch + parameters: + <<: *dummy_stretch_params + sources: &liw_automation_sources + - name: code + repo: git://git.liw.fi/liw-automation + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: liw-automation-sid + parameters: + <<: *dummy_sid_params + sources: *liw_automation_sources + pipelines: *deb_build_pipelines + + + +- project: summain-stretch + parameters: + <<: *dummy_stretch_params + sources: &summain_sources + - name: code + repo: git://git.liw.fi/summain + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: summain-sid + parameters: + <<: *dummy_sid_params + sources: *summain_sources + pipelines: *deb_build_pipelines + + + +- project: apifw-stretch + parameters: + <<: *dummy_stretch_params + sources: &apifw_sources + - name: code + repo: git://git.liw.fi/apifw + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: apifw-sid + parameters: + <<: *dummy_sid_params + sources: *apifw_sources + pipelines: *deb_build_pipelines + + + +- project: slog-stretch + parameters: + <<: *dummy_stretch_params + sources: &slog_sources + - name: code + repo: git://git.liw.fi/slog + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: slog-sid + parameters: + <<: *dummy_sid_params + sources: *slog_sources + pipelines: *deb_build_pipelines + + + +- project: ick2-stretch + parameters: + <<: *dummy_stretch_params + sources: &ick2_sources + - name: code + repo: git://git.liw.fi/ick2 + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: ick2-sid + parameters: + <<: *dummy_sid_params + sources: *ick2_sources + pipelines: *deb_build_pipelines + + + +# No vmdb2 stretch build. It doesn't build there. + +- project: vmdb2-sid + parameters: + <<: *dummy_sid_params + sources: + - name: code + repo: git://git.liw.fi/vmdb2 + ref: master + location: src + pipelines: *deb_build_pipelines + + + +- project: ql-ikiwiki-publish-stretch + parameters: + <<: *dummy_stretch_params + sources: &ql_ikiwiki_publish_sources + - name: code + repo: git://git.qvarnlabs.net/ql-ikiwiki-publish + ref: master + location: src + pipelines: *deb_build_pipelines + +- project: ql-ikiwiki-publish-sid + parameters: + <<: *dummy_sid_params + sources: *ql_ikiwiki_publish_sources + pipelines: *deb_build_pipelines diff --git a/ci-prod-systrees.ick b/ci-prod-systrees.ick new file mode 100644 index 0000000..3683f67 --- /dev/null +++ b/ci-prod-systrees.ick @@ -0,0 +1,74 @@ +projects: + + - project: stretch-base + parameters: &systree_base + debian_codename: stretch + packages: + - jq + - python3 + artifact_name: systrees-debian-stretch-base + notify: [liw@liw.fi] + pipelines: + - ick/build_debian_systree + + - project: stretch-build-essential + parameters: + systree_name: systrees-debian-stretch-base + artifact_name: systrees-debian-stretch-build-essential + packages: + - build-essential + - git + - devscripts + notify: [liw@liw.fi] + pipelines: + - ick/setup_container + - ick/install_debian_packages + - ick/archive_systree + + - project: sid-base + parameters: + <<: *systree_base + debian_codename: sid + artifact_name: systrees-debian-sid-base + pipelines: + - ick/build_debian_systree + + - project: sid-build-essential + parameters: + systree_name: systrees-debian-sid-base + artifact_name: systrees-debian-sid-build-essential + packages: + - build-essential + - git + - devscripts + notify: [liw@liw.fi] + pipelines: + - ick/setup_container + - ick/install_debian_packages + - ick/archive_systree + + # - project: stretch-ikiwiki + # parameters: + # <<: *systree_base + # systree_name: systrees-debian-stretch-base + # artifact_name: systrees-debian-stretch-ikiwiki + # packages: + # - git + # - jq + # - ikiwiki + # - libhighlight-perl + # - libimage-magick-perl + # - graphviz + # - ql-ikiwiki-publish + # - locales-all + # - distix + # - pandoc + # apt_sources: + # - url: http://code.liw.fi/debian + # dist: stretch + # section: main + # pipelines: + # - ick/setup_container + # - ick/add_apt_sources + # - ick/install_debian_packages + # - ick/archive_systree diff --git a/ci-prod-websites.ick b/ci-prod-websites.ick new file mode 100644 index 0000000..6237f1b --- /dev/null +++ b/ci-prod-websites.ick @@ -0,0 +1,186 @@ +projects: + +- project: dummy-ikiwiki + parameters: &ikiwiki_params + rsync_src: html + systree_name: systrees-debian-stretch-distix + notify: [liw@liw.fi] + pipelines: &ikiwiki_pipelines + - ick/setup_container + - ick/get_sources + - ql_ikiwiki_publish + +- project: ick.liw.fi + parameters: + <<: *ikiwiki_params + rsync_target: ickliwfi@pieni.net:/srv/http/ick.liw.fi + sources: + - name: source + ref: master + location: src + repo: git://git.liw.fi/ick.liw.fi + pipelines: *ikiwiki_pipelines + +- project: noir.liw.fi + parameters: + <<: *ikiwiki_params + rsync_target: ickliwfi@pieni.net:/srv/http/noir.liw.fi + sources: + - name: source + ref: master + location: src + repo: git://git.liw.fi/noir.liw.fi + pipelines: *ikiwiki_pipelines + +- project: intra.qvarnlabs.net + parameters: + <<: *ikiwiki_params + notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] + rsync_target: static@static.qvarnlabs.net:/srv/http/intra.qvarnlabs.net + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.qvarnlabs.net/intrawiki + pipelines: *ikiwiki_pipelines + +- project: www.qvarn.org + parameters: + <<: *ikiwiki_params + notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] + rsync_target: static@static.qvarnlabs.net:/srv/http/www.qvarn.org + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.qvarnlabs.net/www.qvarn.org + pipelines: *ikiwiki_pipelines + +- project: www.qvarnlabs.com + parameters: + <<: *ikiwiki_params + notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] + rsync_target: static@static.qvarnlabs.net:/srv/http/www.qvarnlabs.com + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.qvarnlabs.net/www.qvarnlabs.com + pipelines: *ikiwiki_pipelines + +- project: ick-biz.liw.fi + parameters: + <<: *ikiwiki_params + notify: [liw@liw.fi] + rsync_target: ickliwfi@pieni.net:/srv/http/ick-biz.liw.fi + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.liw.fi/ick-biz + pipelines: *ikiwiki_pipelines + + + +- project: ick-yarns.liw.fi + parameters: + sources: + - name: code + ref: master + location: src + repo: git://git.liw.fi/ick2 + rsync_src: html + rsync_target: ickliwfi@pieni.net:/srv/http/ick-yarns.liw.fi + systree_name: systrees-debian-stretch-distix + notify: [liw@liw.fi] + pipelines: + - ick/setup_container + - ick/get_sources + - publish_yarns + +- project: ick-pres.liw.fi + parameters: + sources: + - name: source + ref: master + location: src + repo: ssh://git@git.liw.fi/ick2-pres + rsync_src: html + rsync_target: ickliwfi@pieni.net:/srv/http/ick-pres.liw.fi + systree_name: systrees-debian-stretch-distix + notify: [liw@liw.fi] + pipelines: + - ick/setup_container + - ick/get_sources + - git_export_website + + + +pipelines: + + - pipeline: ql_ikiwiki_publish + parameters: + - sources + - rsync_src + - rsync_target + actions: + - python: | + import os, re, subprocess + def RUN(*args, **kwargs): + print('Executing:', args, kwargs) + subprocess.check_call(args, **kwargs) + + env = dict(os.environ) + env['LC_ALL'] = 'fi_FI.UTF8' + + sources = params['sources'] + for source in sources: + dirname = source['location'] + RUN('rm', '-rf', 'html', 'tmp') + RUN('ql-ikiwiki-preprocess', dirname, 'tmp', env=env) + RUN('sed', '-i', '/^srcdir:/d', 'tmp/ikiwiki.setup', env=env) + RUN('sed', '-i', '/^destdir:/d', 'tmp/ikiwiki.setup', env=env) + with open('tmp/ikiwiki.setup', 'a') as f: + f.write('srcdir: /workspace/tmp\n') + f.write('destdir: /workspace/html\n') + RUN('ikiwiki', '--setup', 'tmp/ikiwiki.setup', '--verbose', env=env) + where: container + + - action: rsync + where: host + + - pipeline: publish_yarns + parameters: + - rsync_src + - rsync_target + actions: + - shell: | + mkdir html + cd src/yarns + pandoc \ + --toc \ + --standalone \ + --self-contained \ + -H yarn.css \ + -o /workspace/html/index.html \ + *.yarn + where: container + + - action: rsync + where: host + + - pipeline: git_export_website + parameters: + - rsync_src + - rsync_target + actions: + + - shell: | + html="$(params | jq -r .rsync_src)" + mkdir "$html" + cd src + git archive HEAD | tar -C "../$html" -xf - + where: container + + - action: rsync + where: host diff --git a/ci-prod.ick b/ci-prod.ick deleted file mode 100644 index c7998aa..0000000 --- a/ci-prod.ick +++ /dev/null @@ -1,1099 +0,0 @@ -# FIXME: -# -# intalling ca-certificates-java runs keytool, which requires /proc to -# be mounted. this is fugly. Some day ick should automatically unmount -# anything mounted into the workspace, and have a declarative way to -# specify mounts. However, ca-certificate-java should not require -# /proc to install. - -projects: - - - project: cachedir - parameters: - sources: - - name: cachedir - repo: git://git.liw.fi/cachedir - ref: master - location: src - systree_name: ick_unstable_systree - notify: [liw@liw.fi] - workspace_name: cachedir_workspace - release_artifact: cachedir_release - pipelines: - - ick/setup_container - - ick/restore_workspace - - ick/get_sources - - ick/install_debian_build_dependencies - - ick/build_deb_ci - - ick/upload_debs - - ick/save_workspace - - # Create systrees for various purposes. I don't have just one, - # minimal systree, and then install any build-dependencies on top of - # that for every build. That would be slow. - - - project: debian_stretch_build_essential_systree - parameters: - debian_codename: stretch - packages: - - git - - build-essential - artifact_name: stretch_build_essential_systree - notify: [liw@liw.fi] - pipelines: - - build_debian_systree - - - project: ikiwiki_systree - parameters: - debian_codename: stretch - packages: - - git - - jq - - ikiwiki - - libhighlight-perl - - libimage-magick-perl - - graphviz - - ql-ikiwiki-publish - - locales-all - - distix - - pandoc - artifact_name: ikiwiki_systree - notify: [liw@liw.fi] - pipelines: - - build_debian_systree - - - project: ick_systree - pipelines: - - build_debian_systree - parameters: - debian_codename: stretch - packages: - - git - - jq - - python3-all - - python3-bottle - - python-cliapp - - python3-cliapp - - python3-coverage-test-runner - - python3-apifw - - python3-slog - - python3-cryptography - - python3-requests - - python-requests - - pycodestyle - - pylint3 - - gunicorn3 - - python3-yaml - - cmdtest - - copyright-statement-lint - - debhelper - - devscripts - artifact_name: ick_systree - notify: [liw@liw.fi] - - - project: ick_unstable_systree - pipelines: - - build_debian_systree - parameters: - debian_codename: stretch - packages: - - git - - jq - - python3-all - - python3-bottle - - python-cliapp - - python3-cliapp - - python3-coverage-test-runner - - python3-apifw - - python3-slog - - python3-cryptography - - python3-requests - - python-requests - - pycodestyle - - pylint3 - - gunicorn3 - - python3-yaml - - cmdtest - - copyright-statement-lint - - debhelper - - devscripts - artifact_name: ick_unstable_systree - notify: [liw@liw.fi] - - - project: qvarndev_systree - pipelines: - - build_debian_systree - parameters: - debian_codename: stretch - packages: - - cmdtest - - copyright-statement-lint - - daemonize - - debhelper - - devscripts - - git - - gunicorn3 - - jq - - pycodestyle - - pylint - - pylint3 - - python-cliapp - - python-crypto - - python-cryptography - - python-jwt - - python-requests - - python3-all - - python3-apifw - - python3-bottle - - python3-cliapp - - python3-coverage-test-runner - - python3-crypto - - python3-cryptography - - python3-jwt - - python3-psycopg2 - - python3-requests - - python3-setuptools - - python3-slog - - python3-ttystatus - - python3-yaml - artifact_name: qvarndev_systree - notify: [liw@liw.fi] - - - project: pythondev_systree - pipelines: - - build_debian_systree - parameters: - debian_codename: stretch - packages: - - jq - - git - - build-essential - - python3-all-dev - - python3-cliapp - - python3-coverage-test-runner - - pycodestyle - - pylint3 - - python3-yaml - - cmdtest - - copyright-statement-lint - - debhelper - - devscripts - artifact_name: pythondev_systree - notify: [liw@liw.fi] - - - project: pythondev_unstable_systree - pipelines: - - build_debian_systree - parameters: - debian_codename: unstable - packages: - - gnupg - - jq - - git - - build-essential - - python3-all-dev - - python3-cliapp - - python3-coverage-test-runner - - pycodestyle - - pylint3 - - python3-yaml - - cmdtest - - copyright-statement-lint - - debhelper - - devscripts - artifact_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - # Build the ick yarns (test suite) as HTML and publish it. - - - project: ick-yarns.liw.fi - parameters: - git_url: git://git.liw.fi/ick2 - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/ick-yarns.liw.fi - systree_name: ikiwiki_systree - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - publish_yarns - - # Build various static websites using ikiwiki, wrapped in a tool to - # add some more functionality. - - - project: ick-biz.liw.fi - parameters: - git_url: ssh://git@git.liw.fi/ick-biz - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/ick-biz.liw.fi - systree_name: ikiwiki_systree - artifact_name: ick-biz.liw.fi-workspace - workspace_name: ick-biz.liw.fi-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - ql_ikiwiki_publish - - persist_workspace - - - project: ick.liw.fi - parameters: - git_url: git://git.liw.fi/ick.liw.fi - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/ick.liw.fi - systree_name: ikiwiki_systree - workspace_name: ick.liw.fi-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - ql_ikiwiki_publish - - persist_workspace - - - project: noir.liw.fi - parameters: - git_url: git://git.liw.fi/noir.liw.fi - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/noir.liw.fi - systree_name: ikiwiki_systree - workspace_name: noir.liw.fi-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - ql_ikiwiki_publish - - persist_workspace - - - project: intra.qvarnlabs.net - parameters: - git_url: ssh://git@git.qvarnlabs.net/intrawiki - git_ref: master - git_dir: src - rsync_src: html - rsync_target: static@static.qvarnlabs.net:/srv/http/intra.qvarnlabs.net - systree_name: ikiwiki_systree - workspace_name: intra.qvarnlabs.net-workspace - notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] - pipelines: - - prepare_container - - prepare_workspace - - ql_ikiwiki_publish - - persist_workspace - - - project: www.qvarn.org - parameters: - git_url: ssh://git@git.qvarnlabs.net/www.qvarn.org - git_ref: master - git_dir: src - rsync_src: html - rsync_target: static@static.qvarnlabs.net:/srv/http/www.qvarn.org - systree_name: ikiwiki_systree - workspace_name: www.qvarn.org-workspace - notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] - pipelines: - - prepare_container - - prepare_workspace - - ql_ikiwiki_publish2 - - persist_workspace - - - project: www.qvarnlabs.com - parameters: - git_url: ssh://git@git.qvarnlabs.net/www.qvarnlabs.com - git_ref: master - git_dir: src - rsync_src: html - rsync_target: static@static.qvarnlabs.net:/srv/http/www.qvarnlabs.com - systree_name: ikiwiki_systree - workspace_name: www.qvarnlabs.com-workspace - notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] - pipelines: - - prepare_container - - prepare_workspace - - ql_ikiwiki_publish - - persist_workspace - - # Build static websites that can be just exported from git. - - - project: ick-pres.liw.fi - parameters: - git_url: ssh://git@git.liw.fi/ick2-pres - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/ick-pres.liw.fi - systree_name: ikiwiki_systree - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - git_export_website - - # Update some static websites from distix ticket repositories. - - - project: bugs@liw.fi - parameters: - git_url: ssh://git@git.liw.fi/bugs-liw-fi-distix - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/bugs.liw.fi - systree_name: ikiwiki_systree - artifact_name: bugs@liw.fi-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - distix_html - - persist_workspace - - - project: tickets@distix.eu - parameters: - git_url: ssh://git@git.liw.fi/distix-bugs - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/tickets.distix.eu - systree_name: ikiwiki_systree - artifact_name: tickets@distix.eu-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - distix_html - - persist_workspace - - - project: ick-support - parameters: - git_url: ssh://git@git.liw.fi/ick-devel-distix - git_ref: master - git_dir: src - rsync_src: html - rsync_target: ickliwfi@pieni.net:/srv/http/ick-support.liw.fi - systree_name: ikiwiki_systree - artifact_name: ick-support-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - distix_html - - persist_workspace - - - project: qvarn-devel - parameters: - git_url: ssh://git@git.qvarnlabs.net/qvarn-devel.distix - git_ref: master - git_dir: src - rsync_src: html - rsync_target: static@static.qvarnlabs.net:/srv/http/qvarn-devel.qvarnlabs.net - systree_name: ikiwiki_systree - artifact_name: qvarn-devel-workspace - notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] - pipelines: - - prepare_container - - prepare_workspace - - distix_html - - persist_workspace - - - project: issues.qvarnlabs.net - parameters: - git_url: ssh://git@git.qvarnlabs.net/issues.distix - git_ref: master - git_dir: src - rsync_src: html - rsync_target: static@static.qvarnlabs.net:/srv/http/issues.qvarnlabs.net - systree_name: ikiwiki_systree - artifact_name: issues.qvarnlabs.net-workspace - notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] - pipelines: - - prepare_container - - prepare_workspace - - distix_html - - persist_workspace - - - project: qvarn-support - parameters: - git_url: ssh://git@git.qvarnlabs.net/qvarn-support.distix - git_ref: master - git_dir: src - rsync_src: html - rsync_target: static@static.qvarnlabs.net:/srv/http/qvarn-support.qvarnlabs.net - systree_name: ikiwiki_systree - artifact_name: qvarn-support-workspace - notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] - pipelines: - - prepare_container - - prepare_workspace - - distix_html - - persist_workspace - - - project: defour-support - parameters: - git_url: ssh://git@git.qvarnlabs.net/defour-support.distix - git_ref: master - git_dir: src - rsync_src: html - rsync_target: static@static.qvarnlabs.net:/srv/http/defour-support.qvarnlabs.net - systree_name: ikiwiki_systree - artifact_name: defour-support-workspace - notify: [liw@qvarnlabs.com, ivan@qvarnlabs.com] - pipelines: - - prepare_container - - prepare_workspace - - distix_html - - persist_workspace - - # Some projects to build Debian packages from source code. The - # packages get uploaeded to the ick instnace's APT repository. - - - project: ick2_unstable - parameters: - git_url: git://git.liw.fi/ick2 - git_ref: master - git_dir: src - package_prefix: "" - systree_name: ick_unstable_systree - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - - project: ick2_release - parameters: - git_url: git://git.liw.fi/ick2 - git_ref: master - git_dir: src - systree_name: ick_systree - notify: [liw@liw.fi] - - workspace_name: ick2_release_workspace - release_artifact: ick2_release - - pipelines: - - prepare_container - - prepare_workspace - - install_deb_build_deps - - build_deb_release - - - project: vmdb2 - parameters: - git_url: git://git.liw.fi/vmdb2 - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - artifact_name: vmdb2-workspace - workspace_name: vmdb2-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - - persist_workspace - - - project: qvarn-jsonb-liwdev - parameters: - git_url: git://git.qvarnlabs.net/qvarn-jsonb - git_ref: liw/dev - git_dir: src - upload_target: liw-ci - package_prefix: "liwdev-" - systree_name: qvarndev_systree - artifact_name: qvarn-jsonb-liwdev-workspace - notify: [liw@liw.fi] - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci_qvarn - - upload_deb - - persist_workspace - - - project: coverage-test-runner - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/coverage-test-runner - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: cliapp - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/cliapp - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: extrautils - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/extrautils - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: cmdtest - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/cmdtest - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: clab - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/clab - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: jt - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/jt - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: bumper - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/bumper - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: py_pgpwordlist - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/py_pgpwordlist - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - - project: distix - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - parameters: - git_url: git://git.liw.fi/distix - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: pythondev_unstable_systree - notify: [liw@liw.fi] - - -# Pipelines used by the projects above. - - -pipelines: - - # Create static HTML from a distix repository. - - - pipeline: distix_html - parameters: - - git_url - - git_ref - - git_dir - - rsync_src - - rsync_target - actions: - - shell: | - git_dir="$(params | jq -r .git_dir)" - cd "$git_dir" - distix html ../html - where: container - - - action: rsync - where: host - - # Build a Debian systree. This is mostly just the debootstrap - # action, but with some juggling to configure APT to use my - # code.liw.fi repository, and installing additional packages. - - - pipeline: build_debian_systree - parameters: - - debian_codename - - packages - - artifact_name - actions: - - debootstrap: auto - mirror: http://deb.debian.org/debian - where: host - - - shell: | - apt-key export code.liw.fi > code.liw.fi.asc - # This is needed for ca-certificates-java - sudo mount --bind /proc proc - where: host - - - shell: | - apt-get install -y gnupg - apt-key add code.liw.fi.asc - cat <<EOF > /etc/apt/sources.list.d/code.liw.fi.list - deb http://code.liw.fi/debian unstable main - EOF - apt-get update - where: chroot - - - shell: | - apt-get install -y python3 - where: chroot - - - python: | - import os, subprocess - def runcmd(argv, **kwargs): - subprocess.check_call(argv, **kwargs) - runcmd(['apt-get', 'install', '-y'] + params['packages']) - where: chroot - - - shell: | - sudo umount proc - where: host - - - archive: workspace - where: host - - # Prepare container for building: get systree from artifact store. - - - pipeline: prepare_container - actions: - - action: populate_systree - where: host - - # Prepare workspace for building: retrieve source code from git. The - # source goes into a subdirectory, mostly because Debian package - # building likes to put things into the parent directory of where - # the build happens. - - - pipeline: prepare_workspace - parameters: - - git_url - - git_ref - - git_dir - actions: - - action: populate_workspace - name_from: workspace_name - where: host - - - action: git - where: host - - - shell: | - dir="$(params | jq -r .git_dir)" - ref="$(params | jq -r .git_ref)" - cd "$dir" - git checkout "$ref" - git merge "origin/$ref" - where: container - - # Build Debian packages. This is a CI build, which means a new - # debian/changelog entry is created, with a version number that - # contains the build number. This way every build results in a new - # package version. - - - pipeline: build_deb_ci - parameters: - - git_url - - git_branch - - upload_target - - package_prefix - actions: - - shell: | - # Remove remnants of Debian packages from previous builds. - rm -f *_* - - missing() { - dpkg-checkbuilddeps 2>&1 | - sed 's/^.*Unmet build dependencies: //' | - tr ' ' '\n' | - grep -v '[()]' - } - - for deb in $(cd src && missing) - do - apt-get install -y "$deb" - done - - T="$(params | jq -r .upload_target)" - PREFIX="$(params | jq -r .package_prefix)" - cd src - case "$PREFIX" in - ?*) - sed -i "/^Source: */s//&${PREFIX}/" debian/control - sed -i "/^Package: */s//&${PREFIX}/" debian/control - ;; - esac - cat debian/control - S="$(dpkg-parsechangelog -S Source)" - U="$(dpkg-parsechangelog -S Version | sed 's/-.*$//')" - D="$(dpkg-parsechangelog -S Version | sed 's/^.*-//')" - U2="${U}.0ci${BUILD_NUMBER}-1" - dch -v "${U2}-1" --package "${PREFIX}${S}" -D "$T" \ - --force-distribution "CI build" - dch -r '' - git archive HEAD | xz > ../"${PREFIX}${S}_${U2}.orig.tar.xz" - dpkg-buildpackage -us -uc - where: container - - # Same as build_deb_ci, but mangle qvarn-common name in - # debian/control dependencies. This is for a work project. - - - pipeline: build_deb_ci_qvarn - parameters: - - git_url - - git_branch - - upload_target - - package_prefix - actions: - - shell: | - # Remove remnants of Debian packages from previous builds. - rm -f *_* - - missing() { - dpkg-checkbuilddeps 2>&1 | - sed 's/^.*Unmet build dependencies: //' | - tr ' ' '\n' | - grep -v '[()]' - } - - for deb in $(cd src && missing) - do - apt-get install -y "$deb" - done - - T="$(params | jq -r .upload_target)" - PREFIX="$(params | jq -r .package_prefix)" - cd src - case "$PREFIX" in - ?*) - sed -i "/^Source: */s//&${PREFIX}/" debian/control - sed -i "/^Package: */s//&${PREFIX}/" debian/control - sed -i "/ \(qvarn-common\)/s// ${PREFIX}\1/" debian/control - ;; - esac - ( - cd debian - for x in qvarn-*.* - do - mv "$x" "${PREFIX}$x" - done - ) - cat debian/control - find debian - S="$(dpkg-parsechangelog -S Source)" - U="$(dpkg-parsechangelog -S Version | sed 's/-.*$//')" - D="$(dpkg-parsechangelog -S Version | sed 's/^.*-//')" - U2="${U}.0ci${BUILD_NUMBER}-1" - dch -v "${U2}-1" --package "${PREFIX}${S}" -D "$T" \ - --force-distribution "CI build" - dch -r '' - git archive HEAD | xz > ../"${PREFIX}${S}_${U2}.orig.tar.xz" - dpkg-buildpackage -us -uc - where: container - - # Upload all built Debian packages. - - - pipeline: upload_deb - actions: - - action: dput - where: host - - # Build a static website using (a wrapper around) ikiwiki. - - - pipeline: ql_ikiwiki_publish - parameters: - - git_dir - - rsync_src - - rsync_target - actions: - - shell: | - export LC_ALL=fi_FI.UTF8 - src="$(params | jq -r .git_dir)" - rm -rf tmp - ql-ikiwiki-preprocess "$src" tmp - sed -i '/^srcdir:/d' tmp/ikiwiki.setup - sed -i '/^destdir:/d' tmp/ikiwiki.setup - echo "srcdir: /workspace/tmp" >> tmp/ikiwiki.setup - echo "destdir: /workspace/html" >> tmp/ikiwiki.setup - cat tmp/ikiwiki.setup - ikiwiki --setup tmp/ikiwiki.setup --verbose - where: container - - - action: rsync - where: host - - # Same as ql_ikiwiki_publish, but diffent repository layout. - - - pipeline: ql_ikiwiki_publish2 - parameters: - - git_dir - - rsync_src - - rsync_target - actions: - - shell: | - export LC_ALL=fi_FI.UTF8 - src="$(params | jq -r .git_dir)" - rm -rf tmp html - ql-ikiwiki-preprocess "$src/src" tmp - sed -i '/^srcdir:/d' tmp/ikiwiki.setup - sed -i '/^destdir:/d' tmp/ikiwiki.setup - echo "srcdir: /workspace/tmp" >> tmp/ikiwiki.setup - echo "destdir: /workspace/html" >> tmp/ikiwiki.setup - mkdir html - cat tmp/ikiwiki.setup - ikiwiki --setup tmp/ikiwiki.setup - find /workspace -ls - where: container - - - action: rsync - where: host - - # Save a copy of the workspace in the artifact store, to allow for - # an incremental build the next time. - - - pipeline: persist_workspace - parameters: - - systree_name - actions: - - archive: workspace - where: host - name_from: workspace_name - - - # Build yarns as HTML and publish them to a site. - - - pipeline: publish_yarns - parameters: - - git_dir - - rsync_src - - rsync_target - actions: - - - shell: | - dpkg -l '*pandoc*' - src="$(params | jq -r .git_dir)" - mkdir html - cd "$src/yarns" - pandoc \ - --toc \ - --standalone \ - --self-contained \ - -H yarn.css \ - -o /workspace/html/index.html \ - *.yarn - where: container - - - action: rsync - where: host - - - # "Publish" contents of a website by exporting from git. - - - pipeline: git_export_website - parameters: - - git_dir - - rsync_src - - rsync_target - actions: - - - shell: | - src="$(params | jq -r .git_dir)" - html="$(params | jq -r .rsync_src)" - - mkdir "$html" - cd "$src" - git archive HEAD | tar -C "../$html" -xf - - where: container - - - action: rsync - where: host - - # Build Debian packages: release build edition. The first time a - # project is built, this pipeline merely remembers all the release - # tags in the repo. Subsequent builds build any new release tags. - # - # The built packages are NOT uploaded to an APT repo. Instead, - # they're put in an artifact, which the user may download, sign, and - # upload to an APT repo themselves. - - - pipeline: install_deb_build_deps - parameters: - - git_dir - actions: - - shell: | - missing() { - dpkg-checkbuilddeps 2>&1 | - sed 's/^.*Unmet build dependencies: //' | - tr ' ' '\n' | - grep -v '[()]' - } - - for deb in $(cd src && missing) - do - apt-get install -y "$deb" - done - where: container - - - pipeline: build_deb_release - parameters: - - git_dir - actions: - - shell: | - list_tags() - { - (cd "$1" && git tag -l) - } - - new_tags() - { - local src="$1" - local list="$2" - - list_tags "$src" | - while read tag - do - if grep -Fx "$tag" "$list" > /dev/null - then - : - else - echo "$tag" - fi - done - } - - build_from_tag() - { - local tag="$1" - - git reset --hard - git clean -fdxq - git checkout "$tag" - - # Remove remnants of Debian packages from previous builds. - rm -f /workspace/*_* - - # Extract source package name and version. - S="$(dpkg-parsechangelog -S Source)" - U="$(dpkg-parsechangelog -S Version | sed 's/-.*$//')" - - # Create upstream tarball. - git archive HEAD | xz > ../"${S}_${U}.orig.tar.xz" - - # ACtually build. - dpkg-buildpackage -us -uc - } - - - SRC="/workspace/$(params | jq -r .git_dir)" - cd "$SRC" - LIST="/workspace/tags.txt" - - if [ -e "$LIST" ] - then - for tag in $(new_tags "$SRC" "$LIST") - do - build_from_tag "$tag" - done - else - echo "No memory of tags" - fi - - # Remember all git tags for future builds. These won't be - # tried again. - echo "Remembering all tags" - list_tags "$SRC" > "$LIST" - where: container - - - archive: workspace - where: host - name_from: workspace_name - - - archive: workspace - where: host - name_from: release_artifact - globs: - - "*_*" - - - # Get an artifact with a persisted workspace and unpack it. Get name - # of artifact from the workspace_name parameter. - - - pipeline: restore_workspace - parameters: - - workspace_name - actions: - - action: populate_workspace - name_from: workspace_name - where: host diff --git a/ickdev.ick b/ickdev.ick deleted file mode 100644 index 13737e0..0000000 --- a/ickdev.ick +++ /dev/null @@ -1,266 +0,0 @@ -# FIXME: -# -# * build dependencies should be installed automatically into container -# * various things should become builtin actions - -projects: - - - project: hello - pipelines: - - hello - parameters: - notify: - - liw@liw.fi - - - project: silent-hello - pipelines: - - hello - - - project: ick_stretch - pipelines: - - build_debian_systree - parameters: - debian_codename: stretch - packages: - - git - - jq - - python3-all - - python3-bottle - - python-cliapp - - python3-cliapp - - python3-coverage-test-runner - - python3-apifw - - python3-slog - - python3-cryptography - - python3-requests - - python-requests - - pycodestyle - - pylint3 - - gunicorn3 - - python3-yaml - - cmdtest - - copyright-statement-lint - systree_name: ick_stretch - artifact_name: ick_stretch - - - project: ick2 - parameters: - git_url: git://git.liw.fi/ick2 - git_ref: master - git_dir: src - systree_name: ick_stretch - workspace_name: ick2_workspace - artifact_name: ick2_workspace - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - - persist_workspace - - - project: rsync - parameters: - systree_name: ick_stretch - rsync_src: html - rsync_target: ickliwfi@pieni.net:test - pipelines: - - make_website - - publish_website - - - project: env - parameters: - systree_name: ick_stretch - pipelines: - - env - -pipelines: - - # Build a Debian systree. This is mostly just the debootstrap - # action, but with some juggling to configure APT. - - - pipeline: build_debian_systree - parameters: - - debian_codename - - packages - - systree_name - - artifact_name - actions: - - debootstrap: stretch - mirror: http://deb.debian.org/debian - where: host - - - shell: | - apt-key export code.liw.fi > code.liw.fi.asc - where: host - - - shell: | - apt-key add code.liw.fi.asc - cat <<EOF > /etc/apt/sources.list.d/code.liw.fi.list - deb http://code.liw.fi/debian unstable main - EOF - apt-get update - where: chroot - - - shell: | - apt-get install -y python3 - where: chroot - - - python: | - import os, subprocess - def runcmd(argv, **kwargs): - subprocess.check_call(argv, **kwargs) - runcmd(['apt-get', 'install', '-y'] + params['packages']) - where: chroot - - - archive: workspace - where: host - - # Prepare container for building: get systree from artifact store, - # copy .ssh from host into workspace, and configure ssh client and - # dput. Also install missing tools. - # - # FIXME: Everything but the systree population should become - # unnecessary. - - - pipeline: prepare_container - parameters: - - systree_name - actions: - - action: populate_systree - systree_name: auto - where: host - - - shell: | - cp -a /home/_ickwm/.ssh . - where: host - - - shell: | - cat <<EOF > .ssh/config - Host * - StrictHostKeyChecking no - UserKnownHostsFile /dev/null - PasswordAuthentication no - KbdInteractiveAuthentication no - EOF - - chown -R root:root .ssh - mv .ssh /root - - export LC_ALL=C - export DEBIAN_FRONTEND=noninteractive - apt install -y debhelper dput devscripts - where: container - - # Prepare workspace for building: retrieve source code from git. The - # source goes into the "src" subdirectory, mostly because Debian - # package building likes to put things into the parent directory of - # where the build happens. - - - pipeline: prepare_workspace - parameters: - - git_url - - git_branch - - git_dir - - workspace_name - actions: - - action: populate_workspace - where: host - - action: git - where: host - - shell: | - dir="$(params | jq -r .git_dir)" - ref="$(params | jq -r .git_ref)" - cd "$dir" - git checkout "$ref" - git merge "origin/$ref" - where: container - - # Store the workspace in the artifact store, for the next build, so - # that the next build doesn't need to start from scratch and git - # clone the whole source tree. - - - pipeline: persist_workspace - parameters: - - artifact_name - actions: - - archive: workspace - where: host - - # Build Debian packages. This is a CI build, which means a new - # debian/changelog entry is created, with a version number that - # contains the build number. This way every build results in a new - # package version. - - - pipeline: build_deb_ci - parameters: - - git_url - - git_branch - actions: - - shell: | - cd src - S="$(dpkg-parsechangelog -S Source)" - U="$(dpkg-parsechangelog -S Version | sed 's/-.*$//')" - D="$(dpkg-parsechangelog -S Version | sed 's/^.*-//')" - U2="${U}.0ci${BUILD_NUMBER}-1" - dch -v "${U2}-1" -D "unstable-ci" --force-distribution "CI build" - dch -r '' - git archive HEAD | xz > ../"${S}_${U2}.orig.tar.xz" - dpkg-buildpackage -us -uc - where: container - - # Upload all built Debian packages. - - - pipeline: upload_deb - actions: - - action: dput - where: host - - # Report environment variables with shell and python actions, on - # host, and in container. - - - pipeline: env - actions: - - shell: | - echo shell env on host - env - where: host - - - shell: | - echo shell env in container - env - where: container - - - python: | - import os - print('python env on host') - for key in os.environ: - print('{}={}'.format(key, os.environ[key])) - where: host - - - python: | - import os - print('python env in container') - for key in os.environ: - print('{}={}'.format(key, os.environ[key])) - where: container - - - - pipeline: make_website - actions: - - shell: | - mkdir -p html - echo "Built by build $BUILD_NUMBER" > html/index.html - where: host - - - pipeline: publish_website - parameters: - - rsync_src - - rsync_target - actions: - - action: rsync - where: host - - - pipeline: hello - actions: - - shell: echo hello, world - where: host diff --git a/testick.ick b/testick.ick deleted file mode 100644 index a8bd4bf..0000000 --- a/testick.ick +++ /dev/null @@ -1,208 +0,0 @@ -projects: - - - project: ick_systree - pipelines: - - build_debian_systree - parameters: - debian_codename: stretch - packages: - - git - - jq - - python3-all - - python3-bottle - - python-cliapp - - python3-cliapp - - python3-coverage-test-runner - - python3-apifw - - python3-slog - - python3-cryptography - - python3-requests - - python-requests - - pycodestyle - - pylint3 - - gunicorn3 - - python3-yaml - - cmdtest - - copyright-statement-lint - - debhelper - - devscripts - artifact_name: ick_systree - - - project: ick2 - parameters: - git_url: git://git.liw.fi/ick2 - git_ref: master - git_dir: src - upload_target: unstable-ci - package_prefix: "" - systree_name: ick_systree - artifact_name: ick2-workspace - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - - persist_workspace - - - project: ick2-liwdev - parameters: - git_url: git://git.liw.fi/ick2 - git_ref: liw/dev - git_dir: src - upload_target: liw-ci - package_prefix: "liwdev-" - systree_name: ick_systree - artifact_name: ick2-liwdev-workspace - pipelines: - - prepare_container - - prepare_workspace - - build_deb_ci - - upload_deb - - persist_workspace - -pipelines: - - # Build a Debian systree. This is mostly just the debootstrap - # action, but with some juggling to configure APT. - - - pipeline: build_debian_systree - parameters: - - debian_codename - - packages - - artifact_name - actions: - - debootstrap: auto - mirror: http://deb.debian.org/debian - where: host - - - shell: | - env - dpkg -l '*gnupg*' - dpkg -S bin/gpg - apt-key export code.liw.fi > code.liw.fi.asc - # This is needed for ca-certificates-java - sudo mount --bind /proc proc - where: host - - - shell: | - apt-get install -y gnupg - env - dpkg -l '*gnupg*' - dpkg -S bin/gpg - apt-key add code.liw.fi.asc - cat <<EOF > /etc/apt/sources.list.d/code.liw.fi.list - deb http://code.liw.fi/debian unstable main - EOF - apt-get update - where: chroot - - - shell: | - apt-get install -y python3 - where: chroot - - - python: | - import os, subprocess - def runcmd(argv, **kwargs): - subprocess.check_call(argv, **kwargs) - runcmd(['apt-get', 'install', '-y'] + params['packages']) - where: chroot - - - shell: | - sudo umount proc - where: host - - - archive: workspace - where: host - - # Prepare container for building: get systree from artifact store, - # copy .ssh from host into workspace, and configure ssh client and - # dput. Also install missing tools. - - - pipeline: prepare_container - actions: - - action: populate_systree - where: host - - # Prepare workspace for building: retrieve source code from git. The - # source goes into the "src" subdirectory, mostly because Debian - # package building likes to put things into the parent directory of - # where the build happens. - - - pipeline: prepare_workspace - parameters: - - git_url - - git_ref - - git_dir - actions: - - action: git - where: host - - shell: | - dir="$(params | jq -r .git_dir)" - ref="$(params | jq -r .git_ref)" - cd "$dir" - git checkout "$ref" - git merge "origin/$ref" - where: container - - # Build Debian packages. This is a CI build, which means a new - # debian/changelog entry is created, with a version number that - # contains the build number. This way every build results in a new - # package version. - - - pipeline: build_deb_ci - parameters: - - git_url - - git_branch - - upload_target - - package_prefix - actions: - - shell: | - # Remove remnants of Debian packages from previous builds. - rm -f *_* - - missing() { - dpkg-checkbuilddeps 2>&1 | - sed 's/^.*Unmet build dependencies: //' | - tr ' ' '\n' | - grep -v '[()]' - } - - for deb in $(cd src && missing) - do - apt-get install -y "$deb" - done - - T="$(params | jq -r .upload_target)" - PREFIX="$(params | jq -r .package_prefix)" - cd src - case "$PREFIX" in - ?*) - sed -i "/^Source: */s//&${PREFIX}/" debian/control - sed -i "/^Package: */s//&${PREFIX}/" debian/control - ;; - esac - cat debian/control - S="$(dpkg-parsechangelog -S Source)" - U="$(dpkg-parsechangelog -S Version | sed 's/-.*$//')" - D="$(dpkg-parsechangelog -S Version | sed 's/^.*-//')" - U2="${U}.0ci${BUILD_NUMBER}-1" - dch -v "${U2}-1" --package "${PREFIX}${S}" -D "$T" \ - --force-distribution "CI build" - dch -r '' - git archive HEAD | xz > ../"${PREFIX}${S}_${U2}.orig.tar.xz" - dpkg-buildpackage -us -uc - where: container - - # Upload all built Debian packages. - - - pipeline: upload_deb - actions: - - action: dput - where: host - - - pipeline: persist_workspace - parameters: - - systree_name - actions: - - archive: workspace - where: host |