gitlab-ci: pre-build containers to improve CI run time and reliability

Shaves about 50% off the build time on both debian and arch builds.
(yeah, I know, it's very small anyway compared to mesa, but we might
 want to add more things in the future)

This also makes the build no longer dependent on external websites:
once the image is build, only fdo-internal services are used.

Signed-off-by: Eric Engestrom <eric.engestrom@intel.com>
Reviewed-by: Michel Dänzer <mdaenzer@redhat.com>

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 47fe3cd..05aa743 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,77 +1,109 @@
-.artifacts-meson: &artifacts-meson
-  when: always
-  paths:
-   - _build/meson-logs
+# This is the tag of the docker image used for the build jobs. If the
+# image doesn't exist yet, the containers stage generates it.
+#
+# In order to generate a new image, one should generally change the tag.
+# While removing the image from the registry would also work, that's not
+# recommended except for ephemeral images during development: Replacing
+# an image after a significant amount of time might pull in newer
+# versions of gcc/clang or other packages, which might break the build
+# with older commits using the same tag.
+#
+# After merging a change resulting in generating a new image to the
+# main repository, it's recommended to remove the image from the source
+# repository's container registry, so that the image from the main
+# repository's registry will be used there as well.
+variables:
+  UPSTREAM_REPO: mesa/drm
+  DEBIAN_TAG: "2019-10-20"
+  DEBIAN_VERSION: buster-slim
+  DEBIAN_IMAGE: "$CI_REGISTRY_IMAGE/debian/$DEBIAN_VERSION:$DEBIAN_TAG"
+  ARCH_TAG: "2019-10-20"
+  ARCH_VERSION: rolling
+  ARCH_IMAGE: "$CI_REGISTRY_IMAGE/archlinux/$ARCH_VERSION:$ARCH_TAG"
 
-.meson-build: &meson-build
-  - meson _build
-      -D amdgpu=true
-      -D cairo-tests=true
-      -D etnaviv=true
-      -D exynos=true
-      -D freedreno=true
-      -D freedreno-kgsl=true
-      -D intel=true
-      -D libkms=true
-      -D man-pages=true
-      -D nouveau=true
-      -D omap=true
-      -D radeon=true
-      -D tegra=true
-      -D udev=true
-      -D valgrind=true
-      -D vc4=true
-      -D vmwgfx=true
-  - ninja -C _build
-  - ninja -C _build test
+include:
+  - project: 'wayland/ci-templates'
+    ref: 1f7f57c64ff4ebbf7292e3b7a13600518b8cb24c
+    file: '/templates/debian.yml'
+  - project: 'wayland/ci-templates'
+    ref: 1f7f57c64ff4ebbf7292e3b7a13600518b8cb24c
+    file: '/templates/arch.yml'
 
-latest-meson:
-  stage: build
-  image: archlinux/base:latest
-  before_script:
-    - pacman -Syu --noconfirm --needed
-        base-devel
-        meson
-        libpciaccess
-        libxslt docbook-xsl
-        valgrind
-        libatomic_ops
-        cairo cunit
-  script: *meson-build
+stages:
+  - containers
+  - build
+
+
+# When & how to run the CI
+.ci-run-policy:
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+
+# CONTAINERS
+
+debian:
+  stage: containers
+  extends:
+    - .ci-run-policy
+    - .debian@container-ifnot-exists
+  variables:
+    GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image
+    DEBIAN_EXEC: 'bash .gitlab-ci/debian-install.sh'
 
-oldest-meson:
+arch:
+  stage: containers
+  extends:
+    - .ci-run-policy
+    - .arch@container-ifnot-exists
+  variables:
+    GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image
+    ARCH_EXEC: 'bash .gitlab-ci/arch-install.sh'
+
+
+# BUILD
+
+.meson-build:
   stage: build
-  image: debian:stable
-  artifacts: *artifacts-meson
-  before_script:
-    - printf > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft "%s\n"
-        'path-exclude=/usr/share/doc/*'
-        'path-exclude=/usr/share/man/*'
-    - printf > /usr/sbin/policy-rc.d "%s\n"
-        '#!/bin/sh'
-        'exit 101'
-    - chmod +x /usr/sbin/policy-rc.d
-    - apt-get update
-    - apt-get -y --no-install-recommends install
-        build-essential
-        pkg-config
-        xsltproc
-        libxslt1-dev docbook-xsl
-        valgrind
-        libatomic-ops-dev
-        libcairo2-dev libcunit1-dev
-        ninja-build
-        python3 python3-pip
-        wget
-    # We need `--no-check-certificate` here because Debian's CA list is
-    # too old to know about LetsEncrypt's CA, so it refuses to connect
-    # to FreeDesktop.org
-    - LIBPCIACCESS_VERSION=libpciaccess-0.10 &&
-      wget --no-check-certificate https://xorg.freedesktop.org/releases/individual/lib/$LIBPCIACCESS_VERSION.tar.bz2 &&
-      tar -jxvf $LIBPCIACCESS_VERSION.tar.bz2 &&
-      (cd $LIBPCIACCESS_VERSION && ./configure --prefix=$HOME/prefix && make install)
-    - pip3 install wheel setuptools
-    - pip3 install meson==0.43
-    - export PKG_CONFIG_PATH=$HOME/prefix/lib/pkgconfig:$HOME/prefix/share/pkgconfig
-    - export LD_LIBRARY_PATH="$HOME/prefix/lib:$LD_LIBRARY_PATH"
-  script: *meson-build
+  extends: .ci-run-policy
+  variables:
+    GIT_DEPTH: 10
+  script:
+    - meson _build
+        -D amdgpu=true
+        -D cairo-tests=true
+        -D etnaviv=true
+        -D exynos=true
+        -D freedreno=true
+        -D freedreno-kgsl=true
+        -D intel=true
+        -D libkms=true
+        -D man-pages=true
+        -D nouveau=true
+        -D omap=true
+        -D radeon=true
+        -D tegra=true
+        -D udev=true
+        -D valgrind=true
+        -D vc4=true
+        -D vmwgfx=true
+    - ninja -C _build
+    - ninja -C _build test
+    - DESTDIR=$PWD/install ninja -C _build install
+  artifacts:
+    when: on_failure
+    paths:
+      - _build/meson-logs/*
+
+meson-debian:
+  extends: .meson-build
+  image: $DEBIAN_IMAGE
+  needs:
+    - debian
+
+meson-arch:
+  extends: .meson-build
+  image: $ARCH_IMAGE
+  needs:
+    - arch

diff --git a/.gitlab-ci/arch-install.sh b/.gitlab-ci/arch-install.sh
new file mode 100644 (file)
index 0000000..7060546
--- /dev/null
+++ b/.gitlab-ci/arch-install.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o xtrace
+
+pacman -Syu --noconfirm --needed \
+  base-devel \
+  cairo \
+  cunit \
+  docbook-xsl \
+  libatomic_ops \
+  libpciaccess \
+  libxslt \
+  meson \
+  valgrind

diff --git a/.gitlab-ci/debian-install.sh b/.gitlab-ci/debian-install.sh
new file mode 100644 (file)
index 0000000..05b63a1
--- /dev/null
+++ b/.gitlab-ci/debian-install.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get install -y \
+  ca-certificates
+
+sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list
+echo 'deb https://deb.debian.org/debian buster-backports main' >/etc/apt/sources.list.d/backports.list
+
+apt-get update
+
+# Use newer packages from backports by default
+cat >/etc/apt/preferences <<EOF
+Package: *
+Pin: release a=buster-backports
+Pin-Priority: 500
+EOF
+
+apt-get dist-upgrade -y
+
+apt-get install -y --no-remove \
+  build-essential \
+  docbook-xsl \
+  libatomic-ops-dev \
+  libcairo2-dev \
+  libcunit1-dev \
+  libpciaccess-dev \
+  libxslt1-dev \
+  ninja-build \
+  pkg-config \
+  python3 \
+  python3-pip \
+  python3-wheel \
+  python3-setuptools \
+  valgrind \
+  xsltproc
+
+
+# Test that the oldest Meson version we claim to support is still supported
+pip3 install meson==0.43