diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a88e22ff50..e0c936e99f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,6 +56,16 @@ variables: # Some jobs may clean up the build artifacts unless this is set to 0. CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 1 + # DNS Shotgun performance testing defaults + SHOTGUN_ROUNDS: 1 + SHOTGUN_DURATION: 120 + # allow unlimited improvements against baseline + SHOTGUN_EVAL_THRESHOLD_CPU_MIN: '-inf' + SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN: '-inf' + SHOTGUN_EVAL_THRESHOLD_RCODE_MAX: '+inf' + SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN: '-inf' + SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN: '-inf' + default: # Allow all running CI jobs to be automatically canceled when a new # version of a branch is pushed. @@ -219,31 +229,18 @@ stages: ### Job Templates .api-pipelines-schedules-tags-triggers-web-triggering-rules: &api_pipelines_schedules_tags_triggers_web_triggering_rules - only: - - api - - pipelines - - schedules - - tags - - triggers - - web + rules: + - if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/' + - if: '$CI_COMMIT_TAG != null' .api-pipelines-schedules-triggers-web-triggering-rules: &api_pipelines_schedules_triggers_web_triggering_rules - only: - - api - - pipelines - - schedules - - triggers - - web + rules: + - if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/' .default-triggering-rules: &default_triggering_rules - only: - - api - - merge_requests - - pipelines - - schedules - - tags - - triggers - - web + rules: + - if: '$CI_PIPELINE_SOURCE =~ /^(api|merge_request_event|pipeline|schedule|trigger|web)$/' + - if: '$CI_COMMIT_TAG != null' .precheck: &precheck_job <<: *default_triggering_rules @@ -343,18 +340,38 @@ stages: .shotgun: &shotgun_job <<: *base_image - <<: *api_pipelines_schedules_tags_triggers_web_triggering_rules stage: performance + rules: + - &shotgun_rule_mr + if: '$CI_MERGE_REQUEST_DIFF_BASE_SHA != null' + variables: + BASELINE: '$CI_MERGE_REQUEST_DIFF_BASE_SHA' + - &shotgun_rule_tag + if: '$CI_COMMIT_TAG != null' + variables: + SHOTGUN_ROUNDS: 3 + - &shotgun_rule_other + if: '$CI_PIPELINE_SOURCE =~ /^(api|pipeline|schedule|trigger|web)$/' script: - - if [ -z "$CI_COMMIT_TAG" ]; then export SHOTGUN_ROUNDS=1; else export SHOTGUN_ROUNDS=3; fi + - if [ -z "$BASELINE" ]; then export BASELINE=$BIND_BASELINE_VERSION; fi # this dotenv variable can't be set in the rules section, because rules are evaluated before any jobs run - PIPELINE_ID=$(curl -s -X POST --fail -F "token=$CI_JOB_TOKEN" -F ref=main - -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BIND_BASELINE_VERSION']" + -F "variables[SHOTGUN_TEST_VERSION]=['$CI_COMMIT_REF_NAME', '$BASELINE']" -F "variables[SHOTGUN_DURATION]=300" -F "variables[SHOTGUN_ROUNDS]=$SHOTGUN_ROUNDS" -F "variables[SHOTGUN_TRAFFIC_MULTIPLIER]=$SHOTGUN_TRAFFIC_MULTIPLIER" -F "variables[SHOTGUN_SCENARIO]=$SHOTGUN_SCENARIO" + -F "variables[SHOTGUN_EVAL_THRESHOLD_CPU_MIN]=$SHOTGUN_EVAL_THRESHOLD_CPU_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_CPU_MAX]=$SHOTGUN_EVAL_THRESHOLD_CPU_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN]=$SHOTGUN_EVAL_THRESHOLD_MEMORY_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_MEMORY_MAX]=$SHOTGUN_EVAL_THRESHOLD_MEMORY_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_RCODE_MIN]=$SHOTGUN_EVAL_THRESHOLD_RCODE_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_RCODE_MAX]=$SHOTGUN_EVAL_THRESHOLD_RCODE_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MIN" + -F "variables[SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX]=$SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_DRIFT_MAX" https://gitlab.isc.org/api/v4/projects/188/trigger/pipeline | jq .id) - util/ci-wait-shotgun.py $PIPELINE_ID needs: @@ -619,9 +636,8 @@ danger: script: - pip install git+https://gitlab.isc.org/isc-projects/hazard.git - hazard - only: - refs: - - merge_requests + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' checkbashisms: <<: *precheck_job @@ -1494,8 +1510,8 @@ release: artifacts: true - job: docs artifacts: true - only: - - tags + rules: + - if: '$CI_COMMIT_TAG != null' artifacts: paths: - "*-release" @@ -1538,8 +1554,8 @@ sign: needs: - job: release artifacts: true - only: - - tags + rules: + - if: '$CI_COMMIT_TAG != null' when: manual allow_failure: false @@ -1591,10 +1607,8 @@ coverity: - cov-int.tar.gz expire_in: "1 week" when: on_failure - only: - variables: - - $COVERITY_SCAN_PROJECT_NAME - - $COVERITY_SCAN_TOKEN + rules: + - if: '$COVERITY_SCAN_PROJECT_NAME != null && $COVERITY_SCAN_TOKEN != null' # Respdiff tests @@ -1670,24 +1684,28 @@ shotgun:tcp: variables: SHOTGUN_SCENARIO: tcp SHOTGUN_TRAFFIC_MULTIPLIER: 13 - when: delayed - start_in: 5 minutes shotgun:dot: <<: *shotgun_job variables: SHOTGUN_SCENARIO: dot SHOTGUN_TRAFFIC_MULTIPLIER: 6 - when: delayed - start_in: 5 minutes + rules: &shotgun_rules_manual_mr + - if: '$CI_MERGE_REQUEST_DIFF_BASE_SHA != null' + variables: + BASELINE: '$CI_MERGE_REQUEST_DIFF_BASE_SHA' + when: manual # don't run on each MR unless requested + allow_failure: true + - *shotgun_rule_tag + - *shotgun_rule_other shotgun:doh-get: <<: *shotgun_job variables: SHOTGUN_SCENARIO: doh-get SHOTGUN_TRAFFIC_MULTIPLIER: 3 - when: delayed - start_in: 5 minutes + SHOTGUN_EVAL_THRESHOLD_LATENCY_PCTL_MAX: 0.3 # bump from the default due to increased tail-end jitter + rules: *shotgun_rules_manual_mr .stress-test: &stress_test stage: performance @@ -1726,8 +1744,8 @@ fsck: - git clone https://gitlab.isc.org/isc-projects/bind9.git bind9-full-clone - cd bind9-full-clone/ - git fsck - only: - - schedules + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' needs: [] gcov: @@ -1779,9 +1797,8 @@ pairwise: - pairwise-model.txt - pairwise-output.*.txt when: on_failure - only: - variables: - - $PAIRWISE_TESTING + rules: + - if: '$PAIRWISE_TESTING != null' .post_merge_template: &post_merge <<: *base_image diff --git a/util/ci-wait-shotgun.py b/util/ci-wait-shotgun.py index c7f5f26eab..894dfa9f57 100755 --- a/util/ci-wait-shotgun.py +++ b/util/ci-wait-shotgun.py @@ -79,13 +79,15 @@ def get_postproc_job(project, pipeline_id): def evaluate_postproc_job(job): - if job.status != "success": - raise RuntimeError("error: 'postproc' job didn't succeed") + print(f"postproc job URL: {job.web_url}") index_url = ( "https://isc-projects.gitlab-pages.isc.org/-/" f"bind9-shotgun-ci/-/jobs/{job.id}/artifacts/index.html" ) - print(f"Result ready for manual inspection: {index_url}") + if job.status in ["success", "failed"]: + print(f"result for manual inspection: {index_url}") + if job.status != "success": + raise RuntimeError("error: 'postproc' job didn't succeed") def main():