Add: ci-prep.ick

1 files changed, 240 insertions, 0 deletions

diff --git a/ci-prep.ick b/ci-prep.ick
new file mode 100644
index 0000000..428bc00
--- /dev/null
+++ b/ci-prep.ick
@@ -0,0 +1,240 @@
+projects:
+
+  - project: ick_systree
+    pipelines:
+      - build_debian_systree
+    parameters:
+      debian_codename: stretch
+      packages:
+        - git
+        - jq
+        - python3-all
+        - python3-bottle
+        - python-cliapp
+        - python3-cliapp
+        - python3-coverage-test-runner
+        - python3-apifw
+        - python3-slog
+        - python3-cryptography
+        - python3-requests
+        - python-requests
+        - pycodestyle
+        - pylint3
+        - gunicorn3
+        - python3-yaml
+        - cmdtest
+        - copyright-statement-lint
+        - debhelper
+        - devscripts
+        - ansible
+      artifact_name: ick_systree
+      notify: [liw@liw.fi]
+
+  - project: ick2
+    parameters:
+      git_url: git://git.liw.fi/ick2
+      git_ref: master
+      git_dir: src
+      upload_target: unstable-ci
+      package_prefix: ""
+      systree_name: ick_systree
+      artifact_name: ick2-workspace
+      notify: [liw@liw.fi]
+    pipelines:
+      - prepare_container
+      # - prepare_workspace
+      # - build_deb_ci
+      # - upload_deb
+      # - persist_workspace
+      - copy_ssh_key
+      - get_ick_playbook
+      - deploy_ick
+
+pipelines:
+
+  # Build a Debian systree. This is mostly just the debootstrap
+  # action, but with some juggling to configure APT.
+
+  - pipeline: build_debian_systree
+    parameters:
+      - debian_codename
+      - packages
+      - artifact_name
+    actions:
+      - debootstrap: auto
+        mirror: http://deb.debian.org/debian
+        where: host
+
+      - shell: |
+          env
+          dpkg -l '*gnupg*'
+          dpkg -S bin/gpg
+          apt-key export code.liw.fi > code.liw.fi.asc
+          # This is needed for ca-certificates-java
+          sudo mount --bind /proc proc
+        where: host
+
+      - shell: |
+          apt-get install -y gnupg
+          env
+          dpkg -l '*gnupg*'
+          dpkg -S bin/gpg
+          apt-key add code.liw.fi.asc
+          cat <<EOF > /etc/apt/sources.list.d/code.liw.fi.list
+          deb http://code.liw.fi/debian unstable main
+          EOF
+          apt-get update
+        where: chroot
+
+      - shell: |
+          apt-get install -y python3
+        where: chroot
+
+      - python: |
+          import os, subprocess
+          def runcmd(argv, **kwargs):
+            subprocess.check_call(argv, **kwargs)
+          runcmd(['apt-get', 'install', '-y'] + params['packages'])
+        where: chroot
+
+      - shell: |
+          sudo umount proc
+        where: host
+
+      - archive: workspace
+        where: host
+
+  # Prepare container for building: get systree from artifact store,
+  # copy .ssh from host into workspace, and configure ssh client and
+  # dput. Also install missing tools.
+
+  - pipeline: prepare_container
+    actions:
+      - action: populate_systree
+        where: host
+
+  # Prepare workspace for building: retrieve source code from git. The
+  # source goes into the "src" subdirectory, mostly because Debian
+  # package building likes to put things into the parent directory of
+  # where the build happens.
+
+  - pipeline: prepare_workspace
+    parameters:
+      - git_url
+      - git_ref
+      - git_dir
+    actions:
+      - action: git
+        where: host
+      - shell: |
+          dir="$(params | jq -r .git_dir)"
+          ref="$(params | jq -r .git_ref)"
+          cd "$dir"
+          git checkout "$ref"
+          git merge "origin/$ref"
+        where: container
+
+  # Build Debian packages. This is a CI build, which means a new
+  # debian/changelog entry is created, with a version number that
+  # contains the build number. This way every build results in a new
+  # package version.
+
+  - pipeline: build_deb_ci
+    parameters:
+      - git_url
+      - git_branch
+      - upload_target
+      - package_prefix
+    actions:
+      - shell: |
+          # Remove remnants of Debian packages from previous builds.
+          rm -f *_*
+
+          missing() {
+            dpkg-checkbuilddeps 2>&1 |
+            sed 's/^.*Unmet build dependencies: //' |
+            tr ' ' '\n' |
+            grep -v '[()]'
+          }
+
+          for deb in $(cd src && missing)
+          do
+              apt-get install -y "$deb"
+          done
+
+          T="$(params | jq -r .upload_target)"
+          PREFIX="$(params | jq -r .package_prefix)"
+          cd src
+          case "$PREFIX" in
+            ?*)
+              sed -i "/^Source: */s//&${PREFIX}/" debian/control
+              sed -i "/^Package: */s//&${PREFIX}/" debian/control
+              ;;
+          esac
+          cat debian/control
+          S="$(dpkg-parsechangelog -S Source)"
+          U="$(dpkg-parsechangelog -S Version | sed 's/-.*$//')"
+          D="$(dpkg-parsechangelog -S Version | sed 's/^.*-//')"
+          U2="${U}.0ci${BUILD_NUMBER}-1"
+          dch -v "${U2}-1" --package "${PREFIX}${S}" -D "$T" \
+            --force-distribution "CI build"
+          dch -r ''
+          git archive HEAD | xz > ../"${PREFIX}${S}_${U2}.orig.tar.xz"
+          dpkg-buildpackage -us -uc
+        where: container
+
+  # Upload all built Debian packages.
+
+  - pipeline: upload_deb
+    actions:
+      - action: dput
+        where: host
+
+  - pipeline: persist_workspace
+    parameters:
+      - systree_name
+    actions:
+      - archive: workspace
+        where: host
+
+  - pipeline: copy_ssh_key
+    actions:
+      - shell: |
+          sudo cp -a /home/_ickwm/.ssh .
+        where: host
+
+  - pipeline: get_ick_playbook
+    actions:
+      - shell: |
+          git clone git://git.liw.fi/ick2-ansible
+          git clone git://git.qvarnlabs.net/debian-ansible
+        where: container
+
+  - pipeline: deploy_ick
+    actions:
+      - shell: |
+          rm -rf /root/.ssh
+          mv .ssh /root/.
+          chown -R root:root /root/.ssh
+          cat << EOF > /root/.ssh/config
+          Host *
+          UserKnownHostsFile /dev/null
+          StrictHostKeyChecking no
+          EOF
+
+          ssh -v git@git.liw.fi whoami
+          git clone ssh://git@git.liw.fi/private/liw/ci-prod-vars
+
+          cd ick2-ansible
+
+          cat > hosts.ci-prod << EOF
+          controller ansible_ssh_host=ickdev-controller.vm.liw.fi
+          worker1 ansible_ssh_host=ickdev-worker1.vm.liw.fi
+          worker2 ansible_ssh_host=ickdev-worker2.vm.liw.fi
+          EOF
+
+          export ANSIBLE_ROLES_PATH=/workspace/debian-ansible/roles
+          ls -l /workspace/ci-prod-vars/ci-prod-vars.yml
+          cat /workspace/ci-prod-vars/ci-prod-vars.yml
+          sh -x ./run-cluster.sh hosts.ci-prod -e @/workspace/ci-prod-vars/ci-prod-vars.yml
+        where: container