From 87d2513823dfca05011bb42f9ff0298931a911bf Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 08:12:41 +0100 Subject: [PATCH 01/11] tests: use one spread worker for ubuntu-cloud-24.04 There's contention between running spread across many nodes, in chunks, in a CI/CD pipeline, and running spread on one machine, across many instances at the same time. The case with CI/CD needs one worker, as parallelism is provided by GitLab. The case with local spread needs many workers as parallelism is provided locally by spread allocating new instances. At present we need to focus on the CI/CD case. I have a plan on how to avoid the problem entirely down the line, by running multiple copies of spread locally, as if everything was done in a CI/CD pipeline. Signed-off-by: Zygmunt Krynicki (cherry picked from commit dfa331dfff3a87472f5d25572792937ec0778a60) Signed-off-by: John Johansen --- spread.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/spread.yaml b/spread.yaml index 586d61d31..ee1ad81ed 100644 --- a/spread.yaml +++ b/spread.yaml @@ -66,7 +66,6 @@ backends: - ubuntu-cloud-24.04: username: ubuntu password: ubuntu - workers: 4 manual: true - ubuntu-cloud-24.10: username: ubuntu From 40510ba5c01d3227e562aee03b0fa15a6270dfde Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 08:10:22 +0100 Subject: [PATCH 02/11] tests: run regression and profile tests with spread This requires a runner with the tags: linux, x86_64, kvm. One needs to be provisioned for the AppArmor project for the pipeline to function. It is possible to run the same tests on SAAS runners offered by GitLab but due to issue gitlab-org/gitlab-runner#6208 there is no way to expose /dev/kvm on the host to the guest. Without this feature emulation works but is rather slow as to be impractical. Note that there's some overlap between the build-all job and spread that might be avoided in the future. At present this is made more difficult by the fact that the path where build-all job builds libapparmor is stored internally by autotools. This prevents us from using GitLab artifacts from moving the built files across to the spread testing jobs without extra work. Signed-off-by: Zygmunt Krynicki (cherry picked from commit 7f68ed174c5a81895c3c19940d7a66211bfaa2b7) Signed-off-by: John Johansen --- .gitlab-ci.yml | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cb107af0b..3527c43f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ workflow: stages: - build - test + - spread .ubuntu-common: before_script: @@ -196,3 +197,106 @@ coverity: - "apparmor-*.tar.gz" rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor" + +.spread: + stage: spread + # TODO: use tagged release once container tagging is improved upstream. + image: registry.gitlab.com/zygoon/image-garden:latest + variables: + ARCH: x86_64 # for cache key :/ + GARDEN_DL_DIR: dl + # GitLab project identifier of zygoon/spread-dist can be seen on + # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on + # top-right. + SPREAD_GITLAB_PROJECT_ID: "65375371" + # Git revision of spread to install. + # This must have been built via spread-dist. + # TODO: switch to upstream 1.0 release when available. + SPREAD_REV: 413817eda7bec07a3885e0717c178b965f8924e1 + # Run all the tasks for a given system. + SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" + before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run + + # Prepare the image, for real. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" + - image-garden make "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image + + # Install the selected revision of spread. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." + # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository. + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --location --output spread "${CI_API_V4_URL}/projects/${SPREAD_GITLAB_PROJECT_ID}/packages/generic/spread/${SPREAD_REV}/spread.${SPREAD_GOARCH}" + - chmod +x spread + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_spread + script: + - printf '\e[0K%s:%s:%s\r\e[0K%s\n' section_start "$(date +%s)" run_spread "Running spread for $GARDEN_SYSTEM..." + # TODO: transform to inject ^...$ to properly select jobs to run. + - mkdir -p spread-logs spread-artifacts + - ./spread -list $SPREAD_ARGS | + split --number=l/${CI_NODE_INDEX:-1}/"${CI_NODE_TOTAL:-1}" | + xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread + cache: + # Cache the base image (pre-customization). + - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH} + policy: $POLICY + when: always + paths: + - $GARDEN_DL_DIR + # Those are never mutated so they are safe to share. + - efi-code.*.img + - efi-vars.*.img + # Cache the customized system. This cache depends on .image-garden.mk file + # so that any customization updates are immediately acted upon. + - key: + prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}- + files: + - .image-garden.mk + policy: $POLICY + when: always + paths: + - $GARDEN_SYSTEM.* + artifacts: + paths: + - spread-logs + - spread-artifacts + when: always + rules: + - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + variables: + # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. + POLICY: pull + - if: $CI_NODE_TOTAL == "1" + variables: + # For sequential jobs we can always push to the cache. + POLICY: pull-push + - if: $CI_NODE_INDEX == "1" + variables: + # For parallel jobs, only the first job pushes to the cache. + POLICY: pull-push + +.spread-x86_64: + extends: .spread + tags: + - linux + - x86_64 + - kvm + variables: + SPREAD_GOARCH: amd64 + ARCH: x86_64 + +spread-ubuntu-cloud-24.04-x86_64: + extends: .spread-x86_64 + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/ + needs: [] + dependencies: [] + parallel: 4 From 1d31e9e3baafc7df011537a7d0f867e9214444b9 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 08:13:48 +0100 Subject: [PATCH 03/11] tests: remove test-build-regression job This job is now redundant since the same operation is done when spread builds and runs regression tests. Signed-off-by: Zygmunt Krynicki (cherry picked from commit f82c8471f5c72e2aabbdbcc5d91c1359da72c2ad) Signed-off-by: John Johansen --- .gitlab-ci.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3527c43f9..2fdb0321b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,19 +127,6 @@ test-profiles: - make -C profiles check-abstractions.d - make -C profiles check-local -# Build the regression tests (don't run them because that needs kernel access) -test-build-regression: - stage: test - needs: ["build-all"] - extends: - - .ubuntu-common - script: - # Additional dependencies required by regression tests - - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_extra_deps "Installing additional dependencies..." - - apt-get install --no-install-recommends -y attr fuse-overlayfs libdbus-1-dev liburing-dev - - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_extra_deps - - make -C tests/regression/apparmor -j $(nproc) - shellcheck: stage: test needs: [] From daaf768b3f9e154e62e789c60ed5678cf0008f69 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 14:40:30 +0100 Subject: [PATCH 04/11] tests: allow non-default branches to push spread cache As a security measure, GitLab splits cache into two broad pools: protected and non-protected. Any job running in a protected branch has access to the protected cache pool. All other jobs run in the non-protected cache pool. This effectively forces us to push to cache in non-protected branches, like all the merge requests, in order to actually use the cache. Ideally we'd disable this protection and only push from the default branch and pull otherwise, as changes to dependency set is rather rare. [1] https://docs.gitlab.com/ee/ci/caching/#use-the-same-cache-for-all-branches Signed-off-by: Zygmunt Krynicki (cherry picked from commit a0adb016317484426c29b474025954c6322bbd89) Signed-off-by: John Johansen --- .gitlab-ci.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fdb0321b..24d161342 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -256,10 +256,19 @@ coverity: - spread-artifacts when: always rules: - - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - variables: - # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. - POLICY: pull + # Due to default cache protection logic in GitLab, pipelines running in + # protected branches (like master in the AppArmor project) do not get + # access, even read access, to protected cache. As such we need to allow + # non-protected branches to push the cache sometimes, or we'd pay the cost + # of never using cache on unprotected branches. + # + # As such disable the first rule below and only consider CI_NODE_TOTAL and + # CI_NODE_INDEX in cache pull/pull-push preference. + # + # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + # variables: + # # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. + # POLICY: pull - if: $CI_NODE_TOTAL == "1" variables: # For sequential jobs we can always push to the cache. @@ -268,6 +277,9 @@ coverity: variables: # For parallel jobs, only the first job pushes to the cache. POLICY: pull-push + - if: $CI_NODE_TOTAL != "1" && $CI_NODE_INDEX != "1" + variables: + POLICY: pull .spread-x86_64: extends: .spread From 21922fea2568e350f9e90d0fc0cff6690bbbd755 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:04:41 +0100 Subject: [PATCH 05/11] tests: improve image caching performance A new explicit, non-parallel job is injected when the .image-garden.mk or .spread.yaml file changes. This job warms up the cache for the subsequent parallel testing jobs. Signed-off-by: Zygmunt Krynicki (cherry picked from commit 14ceb92ca0023ab17c168f173032ed6eec851352) Signed-off-by: John Johansen --- .gitlab-ci.yml | 132 ++++++++++++++++++++++++------------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 24d161342..9c49a89df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,13 +185,50 @@ coverity: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor" -.spread: +.image-garden: stage: spread # TODO: use tagged release once container tagging is improved upstream. image: registry.gitlab.com/zygoon/image-garden:latest variables: ARCH: x86_64 # for cache key :/ GARDEN_DL_DIR: dl + CACHE_POLICY: pull-push + before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run + script: + # Prepare the image, for real. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" + - image-garden make "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image + cache: + # Cache the base image (pre-customization). + - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH} + policy: $CACHE_POLICY + when: always + paths: + - $GARDEN_DL_DIR + # Those are never mutated so they are safe to share. + - efi-code.*.img + - efi-vars.*.img + # Cache the customized system. This cache depends on .image-garden.mk file + # so that any customization updates are immediately acted upon. + - key: + prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}- + files: + - .image-garden.mk + policy: $CACHE_POLICY + when: always + paths: + - $GARDEN_SYSTEM.* + +.spread: + extends: .image-garden + variables: # GitLab project identifier of zygoon/spread-dist can be seen on # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on # top-right. @@ -203,18 +240,6 @@ coverity: # Run all the tasks for a given system. SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" before_script: - # Prepare the image in dry-run mode. This helps in debugging cache misses - # when files are not cached correctly by the runner, causing the build section - # below to always do hevy-duty work. - - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" - - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run - - # Prepare the image, for real. - - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" - - image-garden make "$GARDEN_SYSTEM.$ARCH.run" - - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image - # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository. @@ -230,72 +255,47 @@ coverity: split --number=l/${CI_NODE_INDEX:-1}/"${CI_NODE_TOTAL:-1}" | xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread - cache: - # Cache the base image (pre-customization). - - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH} - policy: $POLICY - when: always - paths: - - $GARDEN_DL_DIR - # Those are never mutated so they are safe to share. - - efi-code.*.img - - efi-vars.*.img - # Cache the customized system. This cache depends on .image-garden.mk file - # so that any customization updates are immediately acted upon. - - key: - prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}- - files: - - .image-garden.mk - policy: $POLICY - when: always - paths: - - $GARDEN_SYSTEM.* artifacts: paths: - spread-logs - spread-artifacts when: always - rules: - # Due to default cache protection logic in GitLab, pipelines running in - # protected branches (like master in the AppArmor project) do not get - # access, even read access, to protected cache. As such we need to allow - # non-protected branches to push the cache sometimes, or we'd pay the cost - # of never using cache on unprotected branches. - # - # As such disable the first rule below and only consider CI_NODE_TOTAL and - # CI_NODE_INDEX in cache pull/pull-push preference. - # - # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - # variables: - # # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. - # POLICY: pull - - if: $CI_NODE_TOTAL == "1" - variables: - # For sequential jobs we can always push to the cache. - POLICY: pull-push - - if: $CI_NODE_INDEX == "1" - variables: - # For parallel jobs, only the first job pushes to the cache. - POLICY: pull-push - - if: $CI_NODE_TOTAL != "1" && $CI_NODE_INDEX != "1" - variables: - POLICY: pull -.spread-x86_64: +# This job builds and caches the image that the job below looks at. +image-ubuntu-cloud-24.04-x86_64: + extends: .image-garden + tags: + - linux + - x86_64 + - kvm + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + ARCH: x86_64 + needs: [] + dependencies: [] + rules: + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH + changes: + paths: + - .image-garden.mk + - spread.yaml + compare_to: "refs/heads/master" + +spread-ubuntu-cloud-24.04-x86_64: extends: .spread tags: - linux - x86_64 - kvm - variables: - SPREAD_GOARCH: amd64 - ARCH: x86_64 - -spread-ubuntu-cloud-24.04-x86_64: - extends: .spread-x86_64 variables: GARDEN_SYSTEM: ubuntu-cloud-24.04 + ARCH: x86_64 + SPREAD_GOARCH: amd64 SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/ - needs: [] + CACHE_POLICY: pull dependencies: [] + needs: + - job: image-ubuntu-cloud-24.04-x86_64 + optional: true parallel: 4 From f255fcfcde81f65b54692c0efe1d53c97d900a07 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:08:31 +0100 Subject: [PATCH 06/11] tests: compress cache faster Our cache is rather compressed already, so this should help a little with wall-clock time. Signed-off-by: Zygmunt Krynicki (cherry picked from commit ebb82952bc7df30cf30db4c937ee1baebfb77b09) Signed-off-by: John Johansen --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c49a89df..e88e52e0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,6 +193,7 @@ coverity: ARCH: x86_64 # for cache key :/ GARDEN_DL_DIR: dl CACHE_POLICY: pull-push + CACHE_COMPRESSION_LEVEL: fastest before_script: # Prepare the image in dry-run mode. This helps in debugging cache misses # when files are not cached correctly by the runner, causing the build section From ef44d8e1777e216dff6c46901b6fc7c4797be80a Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:31:42 +0100 Subject: [PATCH 07/11] tests: reorganize spread pipeline a little This way there's somewhat less repetition and the flow of job definitions is, at least to me, easier to read. Signed-off-by: Zygmunt Krynicki (cherry picked from commit bcf8c968db870fd3dcae1f8987286433f2790ed6) Signed-off-by: John Johansen --- .gitlab-ci.yml | 57 ++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e88e52e0b..c30852564 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,12 +185,16 @@ coverity: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor" -.image-garden: +.image-garden-x86_64: stage: spread # TODO: use tagged release once container tagging is improved upstream. image: registry.gitlab.com/zygoon/image-garden:latest + tags: + - linux + - x86_64 + - kvm variables: - ARCH: x86_64 # for cache key :/ + ARCH: x86_64 GARDEN_DL_DIR: dl CACHE_POLICY: pull-push CACHE_COMPRESSION_LEVEL: fastest @@ -227,8 +231,23 @@ coverity: paths: - $GARDEN_SYSTEM.* -.spread: - extends: .image-garden +# This job builds and caches the image that the job below looks at. +image-ubuntu-cloud-24.04-x86_64: + extends: .image-garden-x86_64 + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + needs: [] + dependencies: [] + rules: + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH + changes: + paths: + - .image-garden.mk + compare_to: "refs/heads/master" + +.spread-x86_64: + extends: .image-garden-x86_64 variables: # GitLab project identifier of zygoon/spread-dist can be seen on # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on @@ -240,6 +259,7 @@ coverity: SPREAD_REV: 413817eda7bec07a3885e0717c178b965f8924e1 # Run all the tasks for a given system. SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" + SPREAD_GOARCH: amd64 before_script: # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." @@ -262,37 +282,10 @@ coverity: - spread-artifacts when: always -# This job builds and caches the image that the job below looks at. -image-ubuntu-cloud-24.04-x86_64: - extends: .image-garden - tags: - - linux - - x86_64 - - kvm - variables: - GARDEN_SYSTEM: ubuntu-cloud-24.04 - ARCH: x86_64 - needs: [] - dependencies: [] - rules: - - if: $CI_COMMIT_TAG - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH - changes: - paths: - - .image-garden.mk - - spread.yaml - compare_to: "refs/heads/master" - spread-ubuntu-cloud-24.04-x86_64: - extends: .spread - tags: - - linux - - x86_64 - - kvm + extends: .spread-x86_64 variables: GARDEN_SYSTEM: ubuntu-cloud-24.04 - ARCH: x86_64 - SPREAD_GOARCH: amd64 SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/ CACHE_POLICY: pull dependencies: [] From 2845e42c5ed5392dac40d2192f4cbbfa09ed786c Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:53:13 +0100 Subject: [PATCH 08/11] tests: quote CI_NODE_INDEX Signed-off-by: Zygmunt Krynicki (cherry picked from commit 62f93b400eda87262ce3f55adbea8a5cefb03001) Signed-off-by: John Johansen --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c30852564..aec3eeed3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -273,7 +273,7 @@ image-ubuntu-cloud-24.04-x86_64: # TODO: transform to inject ^...$ to properly select jobs to run. - mkdir -p spread-logs spread-artifacts - ./spread -list $SPREAD_ARGS | - split --number=l/${CI_NODE_INDEX:-1}/"${CI_NODE_TOTAL:-1}" | + split --number=l/"${CI_NODE_INDEX:-1}"/"${CI_NODE_TOTAL:-1}" | xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread artifacts: From e6d4f79919fca906cab01a8df95473d59c4d3f88 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:53:26 +0100 Subject: [PATCH 09/11] tests: debug image reuse logic We are seeing images cached and then re-constructed as if something had changed in the meanitime. Debug image construction with make --dry-run --debug. Signed-off-by: Zygmunt Krynicki (cherry picked from commit b3ce87af235e3b0730ad807c3dbcfbab27c203ac) Signed-off-by: John Johansen --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aec3eeed3..eceddc83e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -261,6 +261,12 @@ image-ubuntu-cloud-24.04-x86_64: SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" SPREAD_GOARCH: amd64 before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository. From fbb2ae8b058281a0c4d4f1e6ce3eb12d4de6ab47 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:58:54 +0100 Subject: [PATCH 10/11] tests: explicitly cache cloud-init files We were not building or caching the .seed.iso target, causing make to re-create the image, as seen in the make --debug --dry-run output: ``` Updating goal targets.... File ubuntu-cloud-24.04.user-data does not exist. Must remake target ubuntu-cloud-24.04.user-data. echo "${USER_DATA}" | tee ubuntu-cloud-24.04.user-data Successfully remade target file ubuntu-cloud-24.04.user-data. File ubuntu-cloud-24.04.meta-data does not exist. Must remake target ubuntu-cloud-24.04.meta-data. echo "${META_DATA}" | tee ubuntu-cloud-24.04.meta-data Successfully remade target file ubuntu-cloud-24.04.meta-data. Prerequisite ubuntu-cloud-24.04.user-data is newer than target ubuntu-cloud-24.04.seed.iso. Prerequisite ubuntu-cloud-24.04.meta-data is newer than target ubuntu-cloud-24.04.seed.iso. Must remake target ubuntu-cloud-24.04.seed.iso. /usr/bin/genisoimage \ -input-charset utf-8 \ -output ubuntu-cloud-24.04.seed.iso \ -volid CIDATA \ -joliet \ -rock \ -graft-points \ user-data=ubuntu-cloud-24.04.user-data \ meta-data=ubuntu-cloud-24.04.meta-data Successfully remade target file ubuntu-cloud-24.04.seed.iso. Prerequisite ubuntu-cloud-24.04.seed.iso is newer than target ubuntu-cloud-24.04.x86_64.qcow2. ``` Build and cache the cloud-init seed iso to prevent that. Signed-off-by: Zygmunt Krynicki (cherry picked from commit 4cfeb4a9ad3d793302905d2027d96a4c3b3496c7) Signed-off-by: John Johansen --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eceddc83e..5665f110a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,12 +203,12 @@ coverity: # when files are not cached correctly by the runner, causing the build section # below to always do hevy-duty work. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run script: # Prepare the image, for real. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" - - image-garden make "$GARDEN_SYSTEM.$ARCH.run" + - image-garden make "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image cache: # Cache the base image (pre-customization). @@ -230,6 +230,9 @@ coverity: when: always paths: - $GARDEN_SYSTEM.* + - $GARDEN_SYSTEM.seed.iso + - $GARDEN_SYSTEM.meta-data + - $GARDEN_SYSTEM.user-data # This job builds and caches the image that the job below looks at. image-ubuntu-cloud-24.04-x86_64: @@ -244,6 +247,7 @@ image-ubuntu-cloud-24.04-x86_64: changes: paths: - .image-garden.mk + - .gitlab-ci.yml compare_to: "refs/heads/master" .spread-x86_64: @@ -265,7 +269,7 @@ image-ubuntu-cloud-24.04-x86_64: # when files are not cached correctly by the runner, causing the build section # below to always do hevy-duty work. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." From a2335e93952614e92d43042f100227e620e35b58 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 16:19:56 +0100 Subject: [PATCH 11/11] tests: show timestamps of image-garden files Signed-off-by: Zygmunt Krynicki (cherry picked from commit 5a44cbe661509faad08507ec9f4763c45f53cb1a) Signed-off-by: John Johansen --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5665f110a..393f6c72c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -270,6 +270,7 @@ image-ubuntu-cloud-24.04-x86_64: # below to always do hevy-duty work. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" + - stat .image-garden.mk "$GARDEN_SYSTEM".* || true - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..."