From dfa331dfff3a87472f5d25572792937ec0778a60 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 08:12:41 +0100 Subject: [PATCH 01/11] tests: use one spread worker for ubuntu-cloud-24.04 There's contention between running spread across many nodes, in chunks, in a CI/CD pipeline, and running spread on one machine, across many instances at the same time. The case with CI/CD needs one worker, as parallelism is provided by GitLab. The case with local spread needs many workers as parallelism is provided locally by spread allocating new instances. At present we need to focus on the CI/CD case. I have a plan on how to avoid the problem entirely down the line, by running multiple copies of spread locally, as if everything was done in a CI/CD pipeline. Signed-off-by: Zygmunt Krynicki --- spread.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/spread.yaml b/spread.yaml index 556f494c6..ffb4345e9 100644 --- a/spread.yaml +++ b/spread.yaml @@ -66,7 +66,6 @@ backends: - ubuntu-cloud-24.04: username: ubuntu password: ubuntu - workers: 4 manual: true - ubuntu-cloud-24.10: username: ubuntu From 7f68ed174c5a81895c3c19940d7a66211bfaa2b7 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 08:10:22 +0100 Subject: [PATCH 02/11] tests: run regression and profile tests with spread This requires a runner with the tags: linux, x86_64, kvm. One needs to be provisioned for the AppArmor project for the pipeline to function. It is possible to run the same tests on SAAS runners offered by GitLab but due to issue gitlab-org/gitlab-runner#6208 there is no way to expose /dev/kvm on the host to the guest. Without this feature emulation works but is rather slow as to be impractical. Note that there's some overlap between the build-all job and spread that might be avoided in the future. At present this is made more difficult by the fact that the path where build-all job builds libapparmor is stored internally by autotools. This prevents us from using GitLab artifacts from moving the built files across to the spread testing jobs without extra work. Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 621ba2078..ad1780a8f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ workflow: stages: - build - test + - spread .ubuntu-common: before_script: @@ -196,3 +197,106 @@ coverity: - "apparmor-*.tar.gz" rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor" + +.spread: + stage: spread + # TODO: use tagged release once container tagging is improved upstream. + image: registry.gitlab.com/zygoon/image-garden:latest + variables: + ARCH: x86_64 # for cache key :/ + GARDEN_DL_DIR: dl + # GitLab project identifier of zygoon/spread-dist can be seen on + # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on + # top-right. + SPREAD_GITLAB_PROJECT_ID: "65375371" + # Git revision of spread to install. + # This must have been built via spread-dist. + # TODO: switch to upstream 1.0 release when available. + SPREAD_REV: 413817eda7bec07a3885e0717c178b965f8924e1 + # Run all the tasks for a given system. + SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" + before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run + + # Prepare the image, for real. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" + - image-garden make "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image + + # Install the selected revision of spread. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." + # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository. + - | + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --location --output spread "${CI_API_V4_URL}/projects/${SPREAD_GITLAB_PROJECT_ID}/packages/generic/spread/${SPREAD_REV}/spread.${SPREAD_GOARCH}" + - chmod +x spread + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_spread + script: + - printf '\e[0K%s:%s:%s\r\e[0K%s\n' section_start "$(date +%s)" run_spread "Running spread for $GARDEN_SYSTEM..." + # TODO: transform to inject ^...$ to properly select jobs to run. + - mkdir -p spread-logs spread-artifacts + - ./spread -list $SPREAD_ARGS | + split --number=l/${CI_NODE_INDEX:-1}/"${CI_NODE_TOTAL:-1}" | + xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread + cache: + # Cache the base image (pre-customization). + - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH} + policy: $POLICY + when: always + paths: + - $GARDEN_DL_DIR + # Those are never mutated so they are safe to share. + - efi-code.*.img + - efi-vars.*.img + # Cache the customized system. This cache depends on .image-garden.mk file + # so that any customization updates are immediately acted upon. + - key: + prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}- + files: + - .image-garden.mk + policy: $POLICY + when: always + paths: + - $GARDEN_SYSTEM.* + artifacts: + paths: + - spread-logs + - spread-artifacts + when: always + rules: + - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + variables: + # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. + POLICY: pull + - if: $CI_NODE_TOTAL == "1" + variables: + # For sequential jobs we can always push to the cache. + POLICY: pull-push + - if: $CI_NODE_INDEX == "1" + variables: + # For parallel jobs, only the first job pushes to the cache. + POLICY: pull-push + +.spread-x86_64: + extends: .spread + tags: + - linux + - x86_64 + - kvm + variables: + SPREAD_GOARCH: amd64 + ARCH: x86_64 + +spread-ubuntu-cloud-24.04-x86_64: + extends: .spread-x86_64 + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/ + needs: [] + dependencies: [] + parallel: 4 From f82c8471f5c72e2aabbdbcc5d91c1359da72c2ad Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 08:13:48 +0100 Subject: [PATCH 03/11] tests: remove test-build-regression job This job is now redundant since the same operation is done when spread builds and runs regression tests. Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad1780a8f..f71aa1f27 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -127,19 +127,6 @@ test-profiles: - make -C profiles check-abstractions.d - make -C profiles check-local -# Build the regression tests (don't run them because that needs kernel access) -test-build-regression: - stage: test - needs: ["build-all"] - extends: - - .ubuntu-common - script: - # Additional dependencies required by regression tests - - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_extra_deps "Installing additional dependencies..." - - apt-get install --no-install-recommends -y attr fuse-overlayfs libdbus-1-dev liburing-dev - - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" install_extra_deps - - make -C tests/regression/apparmor -j $(nproc) - shellcheck: stage: test needs: [] From a0adb016317484426c29b474025954c6322bbd89 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 14:40:30 +0100 Subject: [PATCH 04/11] tests: allow non-default branches to push spread cache As a security measure, GitLab splits cache into two broad pools: protected and non-protected. Any job running in a protected branch has access to the protected cache pool. All other jobs run in the non-protected cache pool. This effectively forces us to push to cache in non-protected branches, like all the merge requests, in order to actually use the cache. Ideally we'd disable this protection and only push from the default branch and pull otherwise, as changes to dependency set is rather rare. [1] https://docs.gitlab.com/ee/ci/caching/#use-the-same-cache-for-all-branches Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f71aa1f27..7c58f2b67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -256,10 +256,19 @@ coverity: - spread-artifacts when: always rules: - - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - variables: - # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. - POLICY: pull + # Due to default cache protection logic in GitLab, pipelines running in + # protected branches (like master in the AppArmor project) do not get + # access, even read access, to protected cache. As such we need to allow + # non-protected branches to push the cache sometimes, or we'd pay the cost + # of never using cache on unprotected branches. + # + # As such disable the first rule below and only consider CI_NODE_TOTAL and + # CI_NODE_INDEX in cache pull/pull-push preference. + # + # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH + # variables: + # # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. + # POLICY: pull - if: $CI_NODE_TOTAL == "1" variables: # For sequential jobs we can always push to the cache. @@ -268,6 +277,9 @@ coverity: variables: # For parallel jobs, only the first job pushes to the cache. POLICY: pull-push + - if: $CI_NODE_TOTAL != "1" && $CI_NODE_INDEX != "1" + variables: + POLICY: pull .spread-x86_64: extends: .spread From 14ceb92ca0023ab17c168f173032ed6eec851352 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:04:41 +0100 Subject: [PATCH 05/11] tests: improve image caching performance A new explicit, non-parallel job is injected when the .image-garden.mk or .spread.yaml file changes. This job warms up the cache for the subsequent parallel testing jobs. Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 132 ++++++++++++++++++++++++------------------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c58f2b67..f77a4d426 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,13 +185,50 @@ coverity: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor" -.spread: +.image-garden: stage: spread # TODO: use tagged release once container tagging is improved upstream. image: registry.gitlab.com/zygoon/image-garden:latest variables: ARCH: x86_64 # for cache key :/ GARDEN_DL_DIR: dl + CACHE_POLICY: pull-push + before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run + script: + # Prepare the image, for real. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" + - image-garden make "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image + cache: + # Cache the base image (pre-customization). + - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH} + policy: $CACHE_POLICY + when: always + paths: + - $GARDEN_DL_DIR + # Those are never mutated so they are safe to share. + - efi-code.*.img + - efi-vars.*.img + # Cache the customized system. This cache depends on .image-garden.mk file + # so that any customization updates are immediately acted upon. + - key: + prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}- + files: + - .image-garden.mk + policy: $CACHE_POLICY + when: always + paths: + - $GARDEN_SYSTEM.* + +.spread: + extends: .image-garden + variables: # GitLab project identifier of zygoon/spread-dist can be seen on # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on # top-right. @@ -203,18 +240,6 @@ coverity: # Run all the tasks for a given system. SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" before_script: - # Prepare the image in dry-run mode. This helps in debugging cache misses - # when files are not cached correctly by the runner, causing the build section - # below to always do hevy-duty work. - - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" - - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run - - # Prepare the image, for real. - - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" - - image-garden make "$GARDEN_SYSTEM.$ARCH.run" - - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image - # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository. @@ -230,72 +255,47 @@ coverity: split --number=l/${CI_NODE_INDEX:-1}/"${CI_NODE_TOTAL:-1}" | xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread - cache: - # Cache the base image (pre-customization). - - key: image-garden-base-${GARDEN_SYSTEM}.${ARCH} - policy: $POLICY - when: always - paths: - - $GARDEN_DL_DIR - # Those are never mutated so they are safe to share. - - efi-code.*.img - - efi-vars.*.img - # Cache the customized system. This cache depends on .image-garden.mk file - # so that any customization updates are immediately acted upon. - - key: - prefix: image-garden-custom-${GARDEN_SYSTEM}.${ARCH}- - files: - - .image-garden.mk - policy: $POLICY - when: always - paths: - - $GARDEN_SYSTEM.* artifacts: paths: - spread-logs - spread-artifacts when: always - rules: - # Due to default cache protection logic in GitLab, pipelines running in - # protected branches (like master in the AppArmor project) do not get - # access, even read access, to protected cache. As such we need to allow - # non-protected branches to push the cache sometimes, or we'd pay the cost - # of never using cache on unprotected branches. - # - # As such disable the first rule below and only consider CI_NODE_TOTAL and - # CI_NODE_INDEX in cache pull/pull-push preference. - # - # - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH - # variables: - # # Dependencies change rarely so not having to worry about pushes from other branches removes complexity. - # POLICY: pull - - if: $CI_NODE_TOTAL == "1" - variables: - # For sequential jobs we can always push to the cache. - POLICY: pull-push - - if: $CI_NODE_INDEX == "1" - variables: - # For parallel jobs, only the first job pushes to the cache. - POLICY: pull-push - - if: $CI_NODE_TOTAL != "1" && $CI_NODE_INDEX != "1" - variables: - POLICY: pull -.spread-x86_64: +# This job builds and caches the image that the job below looks at. +image-ubuntu-cloud-24.04-x86_64: + extends: .image-garden + tags: + - linux + - x86_64 + - kvm + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + ARCH: x86_64 + needs: [] + dependencies: [] + rules: + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH + changes: + paths: + - .image-garden.mk + - spread.yaml + compare_to: "refs/heads/master" + +spread-ubuntu-cloud-24.04-x86_64: extends: .spread tags: - linux - x86_64 - kvm - variables: - SPREAD_GOARCH: amd64 - ARCH: x86_64 - -spread-ubuntu-cloud-24.04-x86_64: - extends: .spread-x86_64 variables: GARDEN_SYSTEM: ubuntu-cloud-24.04 + ARCH: x86_64 + SPREAD_GOARCH: amd64 SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/ - needs: [] + CACHE_POLICY: pull dependencies: [] + needs: + - job: image-ubuntu-cloud-24.04-x86_64 + optional: true parallel: 4 From ebb82952bc7df30cf30db4c937ee1baebfb77b09 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:08:31 +0100 Subject: [PATCH 06/11] tests: compress cache faster Our cache is rather compressed already, so this should help a little with wall-clock time. Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f77a4d426..aee5308d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,6 +193,7 @@ coverity: ARCH: x86_64 # for cache key :/ GARDEN_DL_DIR: dl CACHE_POLICY: pull-push + CACHE_COMPRESSION_LEVEL: fastest before_script: # Prepare the image in dry-run mode. This helps in debugging cache misses # when files are not cached correctly by the runner, causing the build section From bcf8c968db870fd3dcae1f8987286433f2790ed6 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:31:42 +0100 Subject: [PATCH 07/11] tests: reorganize spread pipeline a little This way there's somewhat less repetition and the flow of job definitions is, at least to me, easier to read. Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 57 ++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aee5308d3..2c143b5d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,12 +185,16 @@ coverity: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "apparmor/apparmor" -.image-garden: +.image-garden-x86_64: stage: spread # TODO: use tagged release once container tagging is improved upstream. image: registry.gitlab.com/zygoon/image-garden:latest + tags: + - linux + - x86_64 + - kvm variables: - ARCH: x86_64 # for cache key :/ + ARCH: x86_64 GARDEN_DL_DIR: dl CACHE_POLICY: pull-push CACHE_COMPRESSION_LEVEL: fastest @@ -227,8 +231,23 @@ coverity: paths: - $GARDEN_SYSTEM.* -.spread: - extends: .image-garden +# This job builds and caches the image that the job below looks at. +image-ubuntu-cloud-24.04-x86_64: + extends: .image-garden-x86_64 + variables: + GARDEN_SYSTEM: ubuntu-cloud-24.04 + needs: [] + dependencies: [] + rules: + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH + changes: + paths: + - .image-garden.mk + compare_to: "refs/heads/master" + +.spread-x86_64: + extends: .image-garden-x86_64 variables: # GitLab project identifier of zygoon/spread-dist can be seen on # https://gitlab.com/zygoon/spread-dist, under the three-dot menu on @@ -240,6 +259,7 @@ coverity: SPREAD_REV: 413817eda7bec07a3885e0717c178b965f8924e1 # Run all the tasks for a given system. SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" + SPREAD_GOARCH: amd64 before_script: # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." @@ -262,37 +282,10 @@ coverity: - spread-artifacts when: always -# This job builds and caches the image that the job below looks at. -image-ubuntu-cloud-24.04-x86_64: - extends: .image-garden - tags: - - linux - - x86_64 - - kvm - variables: - GARDEN_SYSTEM: ubuntu-cloud-24.04 - ARCH: x86_64 - needs: [] - dependencies: [] - rules: - - if: $CI_COMMIT_TAG - - if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH - changes: - paths: - - .image-garden.mk - - spread.yaml - compare_to: "refs/heads/master" - spread-ubuntu-cloud-24.04-x86_64: - extends: .spread - tags: - - linux - - x86_64 - - kvm + extends: .spread-x86_64 variables: GARDEN_SYSTEM: ubuntu-cloud-24.04 - ARCH: x86_64 - SPREAD_GOARCH: amd64 SPREAD_ARGS: garden:$GARDEN_SYSTEM:tests/regression/ garden:$GARDEN_SYSTEM:tests/profiles/ CACHE_POLICY: pull dependencies: [] From 62f93b400eda87262ce3f55adbea8a5cefb03001 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:53:13 +0100 Subject: [PATCH 08/11] tests: quote CI_NODE_INDEX Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c143b5d1..e0e67a019 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -273,7 +273,7 @@ image-ubuntu-cloud-24.04-x86_64: # TODO: transform to inject ^...$ to properly select jobs to run. - mkdir -p spread-logs spread-artifacts - ./spread -list $SPREAD_ARGS | - split --number=l/${CI_NODE_INDEX:-1}/"${CI_NODE_TOTAL:-1}" | + split --number=l/"${CI_NODE_INDEX:-1}"/"${CI_NODE_TOTAL:-1}" | xargs --verbose ./spread -v -artifacts ./spread-artifacts -v | tee spread-logs/"$GARDEN_SYSTEM".log - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" run_spread artifacts: From b3ce87af235e3b0730ad807c3dbcfbab27c203ac Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:53:26 +0100 Subject: [PATCH 09/11] tests: debug image reuse logic We are seeing images cached and then re-constructed as if something had changed in the meanitime. Debug image construction with make --dry-run --debug. Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e0e67a019..242381bd7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -261,6 +261,12 @@ image-ubuntu-cloud-24.04-x86_64: SPREAD_ARGS: "garden:$GARDEN_SYSTEM:" SPREAD_GOARCH: amd64 before_script: + # Prepare the image in dry-run mode. This helps in debugging cache misses + # when files are not cached correctly by the runner, causing the build section + # below to always do hevy-duty work. + - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." # Install pre-built spread from https://gitlab.com/zygoon/spread-dist generic package repository. From 4cfeb4a9ad3d793302905d2027d96a4c3b3496c7 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 15:58:54 +0100 Subject: [PATCH 10/11] tests: explicitly cache cloud-init files We were not building or caching the .seed.iso target, causing make to re-create the image, as seen in the make --debug --dry-run output: ``` Updating goal targets.... File ubuntu-cloud-24.04.user-data does not exist. Must remake target ubuntu-cloud-24.04.user-data. echo "${USER_DATA}" | tee ubuntu-cloud-24.04.user-data Successfully remade target file ubuntu-cloud-24.04.user-data. File ubuntu-cloud-24.04.meta-data does not exist. Must remake target ubuntu-cloud-24.04.meta-data. echo "${META_DATA}" | tee ubuntu-cloud-24.04.meta-data Successfully remade target file ubuntu-cloud-24.04.meta-data. Prerequisite ubuntu-cloud-24.04.user-data is newer than target ubuntu-cloud-24.04.seed.iso. Prerequisite ubuntu-cloud-24.04.meta-data is newer than target ubuntu-cloud-24.04.seed.iso. Must remake target ubuntu-cloud-24.04.seed.iso. /usr/bin/genisoimage \ -input-charset utf-8 \ -output ubuntu-cloud-24.04.seed.iso \ -volid CIDATA \ -joliet \ -rock \ -graft-points \ user-data=ubuntu-cloud-24.04.user-data \ meta-data=ubuntu-cloud-24.04.meta-data Successfully remade target file ubuntu-cloud-24.04.seed.iso. Prerequisite ubuntu-cloud-24.04.seed.iso is newer than target ubuntu-cloud-24.04.x86_64.qcow2. ``` Build and cache the cloud-init seed iso to prevent that. Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 242381bd7..2b4091173 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,12 +203,12 @@ coverity: # when files are not cached correctly by the runner, causing the build section # below to always do hevy-duty work. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run script: # Prepare the image, for real. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image "Prepare image" - - image-garden make "$GARDEN_SYSTEM.$ARCH.run" + - image-garden make "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image cache: # Cache the base image (pre-customization). @@ -230,6 +230,9 @@ coverity: when: always paths: - $GARDEN_SYSTEM.* + - $GARDEN_SYSTEM.seed.iso + - $GARDEN_SYSTEM.meta-data + - $GARDEN_SYSTEM.user-data # This job builds and caches the image that the job below looks at. image-ubuntu-cloud-24.04-x86_64: @@ -244,6 +247,7 @@ image-ubuntu-cloud-24.04-x86_64: changes: paths: - .image-garden.mk + - .gitlab-ci.yml compare_to: "refs/heads/master" .spread-x86_64: @@ -265,7 +269,7 @@ image-ubuntu-cloud-24.04-x86_64: # when files are not cached correctly by the runner, causing the build section # below to always do hevy-duty work. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" + - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..." From 5a44cbe661509faad08507ec9f4763c45f53cb1a Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Thu, 30 Jan 2025 16:19:56 +0100 Subject: [PATCH 11/11] tests: show timestamps of image-garden files Signed-off-by: Zygmunt Krynicki --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b4091173..2700a4ad0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -270,6 +270,7 @@ image-ubuntu-cloud-24.04-x86_64: # below to always do hevy-duty work. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" prepare_image_dry_run "Prepare image (dry run)" - image-garden make --dry-run --debug "$GARDEN_SYSTEM.$ARCH.run" "$GARDEN_SYSTEM.$ARCH.qcow2" "$GARDEN_SYSTEM.seed.iso" "$GARDEN_SYSTEM.user-data" "$GARDEN_SYSTEM.meta-data" + - stat .image-garden.mk "$GARDEN_SYSTEM".* || true - printf '\e[0K%s:%s:%s\r\e[0K\n' section_end "$(date +%s)" prepare_image_dry_run # Install the selected revision of spread. - printf '\e[0K%s:%s:%s[collapsed=true]\r\e[0K%s\n' section_start "$(date +%s)" install_spread "Installing spread..."