tests: unify CI/CD preparation phase

We now have GitLab CI/CD pipeline co-existing with spread, coupled with
image-garden and the cloud-init profile defined for each distribution.

To avoid duplicating list of required dependencies, re-use cloud-init
profile as the reference list of dependencies (superset between build
and test) to install.

In addition to the dependency list, the build_all job now re-uses spread
prepare section in similar fashion. If it builds in spread, it should
build in CI as well.

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>

.gitlab-ci.yml

@@ -14,21 +14,28 @@ stages:
   - build
   - test
 
-.ubuntu-before_script:
+.ubuntu-common:
   before_script:
-    - export DEBIAN_FRONTEND=noninteractive
+    # Install build-dependencies by loading the package list from the ubuntu/debian cloud-init profile.
     - apt-get update -qq
-    - apt-get install --no-install-recommends -y gcc perl liblocale-gettext-perl linux-libc-dev lsb-release make
+    - apt-get install --yes yq make lsb-release
+    - |
+      printf 'include .image-garden.mk\n$(info $(UBUNTU_CLOUD_INIT_USER_DATA_TEMPLATE))\n.PHONY: nothing\nnothing:\n' \
+      | make -f - nothing \
+      | yq '.packages | .[]' \
+      | xargs apt-get install --yes --no-install-recommends
+  after_script:
+    # Inspect the kernel and lsb-release.
     - lsb_release -a
     - uname -a
 
-.install-c-build-deps: &install-c-build-deps
-  - apt-get install --no-install-recommends -y build-essential apache2-dev autoconf autoconf-archive automake bison dejagnu flex libpam-dev libtool pkg-config python3-all-dev python3-setuptools ruby-dev swig zlib1g-dev
-
 build-all:
   stage: build
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
+  script:
+    # Run the spread prepare section to build everything.
+    - yq -r '.prepare' <spread.yaml | SPREAD_PATH=. bash -xeu
   artifacts:
     name: ${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA}
     expire_in: 30 days
@@ -41,23 +48,13 @@ build-all:
       - changehat/mod_apparmor/
       - changehat/pam_apparmor/
       - profiles/
-  script:
-    - *install-c-build-deps
-    - cd libraries/libapparmor && ./autogen.sh && ./configure --with-perl --with-python --prefix=/usr && make -j $(nproc) && cd ../.. || { cat config.log ; exit 1 ; }
-    - make -C parser -j $(nproc)
-    - make -C binutils -j $(nproc)
-    - make -C utils
-    - make -C changehat/mod_apparmor
-    - make -C changehat/pam_apparmor
-    - make -C profiles
 
 test-libapparmor:
   stage: test
   needs: ["build-all"]
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
-    - *install-c-build-deps
     # This is to touch the built files in the test stage to avoid needless rebuilding
     - make -C libraries/libapparmor --touch
     - make -C libraries/libapparmor check
@@ -66,9 +63,8 @@ test-parser:
   stage: test
   needs: ["build-all"]
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
-    - *install-c-build-deps
     # This is to touch the built files in the test stage to avoid needless rebuilding
     - make -C parser --touch
     - make -C parser -j $(nproc) tst_binaries
@@ -78,7 +74,7 @@ test-binutils:
   stage: test
   needs: ["build-all"]
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
     - make -C binutils check
 
@@ -86,11 +82,12 @@ test-utils:
   stage: test
   needs: ["build-all"]
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
     # This is to touch the built files in the test stage to avoid needless rebuilding
     - make -C utils --touch
 
+    # TODO: move those to cloud-init list?
     - apt-get install --no-install-recommends -y libc6-dev libjs-jquery libjs-jquery-throttle-debounce libjs-jquery-isonscreen libjs-jquery-tablesorter flake8 python3-coverage python3-notify2 python3-psutil python3-setuptools python3-tk python3-ttkthemes python3-gi
 
     # See apparmor/apparmor#221
@@ -107,7 +104,7 @@ test-mod-apparmor:
   stage: test
   needs: ["build-all"]
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
     # This is to touch the built files in the test stage to avoid needless rebuilding
     - make -C changehat/mod_apparmor --touch
@@ -117,7 +114,7 @@ test-profiles:
   stage: test
   needs: ["build-all"]
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
     # This is to touch the built files in the test stage to avoid needless rebuilding
     - make -C profiles --touch
@@ -130,17 +127,17 @@ test-build-regression:
   stage: test
   needs: ["build-all"]
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
-    - *install-c-build-deps
+    # Additional dependencies required by regression tests
-      # Additional dependencies required by regression tests
     - apt-get install --no-install-recommends -y attr libdbus-1-dev liburing-dev
     - make -C tests/regression/apparmor -j $(nproc)
+
 shellcheck:
   stage: test
   needs: []
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
   - apt-get install --no-install-recommends -y python3-minimal file shellcheck xmlstarlet
   - shellcheck --version
@@ -171,10 +168,9 @@ variables:
 coverity:
   stage: .post
   extends:
-    - .ubuntu-before_script
+    - .ubuntu-common
   script:
       - apt-get install --no-install-recommends -y curl git texlive-latex-recommended
-      - *install-c-build-deps
       - curl -o /tmp/cov-analysis-linux64.tgz https://scan.coverity.com/download/linux64
         --form project=$COVERITY_SCAN_PROJECT_NAME --form token=$COVERITY_SCAN_TOKEN
       - tar xfz /tmp/cov-analysis-linux64.tgz

spread.yaml

@@ -102,18 +102,26 @@ prepare: |
     # invocations of make below are efficient if nothing needs to be done.
     if [ ! -f "$SPREAD_PATH"/libraries/libapparmor/Makefile ]; then
       (
-        cd $SPREAD_PATH/libraries/libapparmor
+        cd "$SPREAD_PATH"/libraries/libapparmor || exit 1
-        sh ./autogen.sh && sh ./configure --prefix=/usr --with-perl --with-python
+        if ! sh ./autogen.sh; then
+            echo "The autogen.sh script has failed"
+            exit 1
+        fi
+        if ! sh ./configure --prefix=/usr --with-perl --with-python; then
+            echo "The generated configure script has failed"
+            cat config.log
+            exit 1
+        fi
       )
     fi
     # Build libapparmor.
-    make -C $SPREAD_PATH/libraries/libapparmor -j"$(nproc)"
+    make -C "$SPREAD_PATH"/libraries/libapparmor -j"$(nproc)"
     # Build apparmor_parser.
-    make -C $SPREAD_PATH/parser -j"$(nproc)"
+    make -C "$SPREAD_PATH"/parser -j"$(nproc)"
     # Build binary utilities (aa-exec and firends).
-    make -C $SPREAD_PATH/binutils -j"$(nproc)"
+    make -C "$SPREAD_PATH"/binutils -j"$(nproc)"
     # Build python utilities.
-    make -C $SPREAD_PATH/utils -j"$(nproc)"
+    make -C "$SPREAD_PATH"/utils -j"$(nproc)"
     # Build apache and pam modules.
     make -C "$SPREAD_PATH"/changehat/mod_apparmor -j"$(nproc)"
     make -C "$SPREAD_PATH"/changehat/pam_apparmor -j"$(nproc)"