diff --git a/.dockerignore b/.dockerignore index 6882e4713..995a18abd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ **/.github **/_template img/ +**/.env diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e7fd66eb..e40c6aef7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,10 @@ -# Much copied from sbt-github-actions, modified to support running e2e tests name: Continuous Integration on: pull_request: - branches: ['*'] + branches: [ '*' ] push: - branches: ['master','main'] + branches: [ 'master','main' ] env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -13,131 +12,20 @@ env: jobs: build: name: Build and Test + runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'ci skip')" - strategy: - matrix: - os: [ubuntu-latest] - scala: [2.12.10] - java: [adopt@1.11] - runs-on: ${{ matrix.os }} + steps: - name: Checkout current branch (full) uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Setup Java and Scala - uses: olafurpg/setup-scala@v10 - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - with: - java-version: ${{ matrix.java }} - - - name: Cache ivy2 - uses: actions/cache@v1 - with: - path: ~/.ivy2/cache - key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (generic) - uses: actions/cache@v1 - with: - path: ~/.coursier/cache/v1 - key: ${{ runner.os }}-generic-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (linux) - if: contains(runner.os, 'linux') - uses: actions/cache@v1 - with: - path: ~/.cache/coursier/v1 - key: ${{ runner.os }}-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (macOS) - if: contains(runner.os, 'macos') - uses: actions/cache@v1 - with: - path: ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (windows) - if: contains(runner.os, 'windows') - uses: actions/cache@v1 - with: - path: ~/AppData/Local/Coursier/Cache/v1 - key: ${{ runner.os }}-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache sbt - uses: actions/cache@v1 - with: - path: ~/.sbt - key: ${{ runner.os }}-sbt-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - run: sbt ++${{ matrix.scala }} validate verify + - name: Import Content Trust Key + run: cd build/ && ./assemble_api.jar.sh && ./prepare_release.sh + shell: bash - name: Codecov uses: codecov/codecov-action@v1 with: fail_ci_if_error: true # optional (default = false) - - func: - name: Func Test - if: "!contains(github.event.head_commit.message, 'ci skip')" - strategy: - matrix: - os: [ubuntu-latest] - scala: [2.12.10] - java: [adopt@1.11] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout current branch (full) - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Setup Java and Scala - uses: olafurpg/setup-scala@v10 - env: - ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' - with: - java-version: ${{ matrix.java }} - - - name: Cache ivy2 - uses: actions/cache@v1 - with: - path: ~/.ivy2/cache - key: ${{ runner.os }}-sbt-ivy-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (generic) - uses: actions/cache@v1 - with: - path: ~/.coursier/cache/v1 - key: ${{ runner.os }}-generic-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (linux) - if: contains(runner.os, 'linux') - uses: actions/cache@v1 - with: - path: ~/.cache/coursier/v1 - key: ${{ runner.os }}-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (macOS) - if: contains(runner.os, 'macos') - uses: actions/cache@v1 - with: - path: ~/Library/Caches/Coursier/v1 - key: ${{ runner.os }}-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache coursier (windows) - if: contains(runner.os, 'windows') - uses: actions/cache@v1 - with: - path: ~/AppData/Local/Coursier/Cache/v1 - key: ${{ runner.os }}-sbt-coursier-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - - name: Cache sbt - uses: actions/cache@v1 - with: - path: ~/.sbt - key: ${{ runner.os }}-sbt-cache-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }} - - name: Func tests - run: ./utils/func-test-portal.sh && ./utils/func-test-api.sh diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a9bdd702b..b8087c285 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -66,11 +66,9 @@ The API is the RESTful API for interacting with VinylDNS. The following technolo * [Akka HTTP](https://doc.akka.io/docs/akka-http/current/) - Used primarily for REST and HTTP calls. * [FS2](https://functional-streams-for-scala.github.io/fs2/) - Used for backend change processing off of message queues. FS2 has back-pressure built in, and gives us tools like throttling and concurrency. -* [Cats Effect](https://typelevel.org/cats-effect/) - We are currently migrating away from `Future` as our primary type - and towards cats effect IO. Hopefully, one day, all the things will be using IO. +* [Cats Effect](https://typelevel.org/cats-effect/) - A replacement of `Future` with the `IO` monad * [Cats](https://typelevel.org/cats) - Used for functional programming. -* [PureConfig](https://pureconfig.github.io/) - For loading configuration values. We are currently migrating to use - PureConfig everywhere. Not all the places use it yet. +* [PureConfig](https://pureconfig.github.io/) - For loading configuration values. The API has the following dependencies: @@ -137,7 +135,7 @@ settings for the microsite are also configured in `build.sbt` of the project roo #### Code Layout * `src/main/resources` - Microsite resources and configurations -* `src/main/tut` - Content for microsite web pages +* `src/main/mdoc` - Content for microsite web pages ## Running VinylDNS Locally @@ -168,10 +166,11 @@ configuration. ### Starting the Portal -To run the portal locally, you _first_ have to start up the VinylDNS API Server: +To run the portal locally, you _first_ have to start up the VinylDNS API Server. This can be done by following the +instructions for [Staring the API Server](#Starting the API Server) or by using the QuickStart: ``` -utils/quickstart-vinyldns.sh +quickstart/quickstart-vinyldns.sh --api-only ``` Once that is done, in the same `sbt` session or a different one, go to `project portal` and then @@ -333,4 +332,4 @@ zone_id = shared_zone_test_context.ok_zone["id"] When functional tests are run, we spin up several Docker containers. One of the Docker containers is a Bind9 DNS server. If you need to add or modify the test DNS zone files, you can find them in -`docker/bind9/zones` +`quickstart/bind9/zones` diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 88f3b9e72..40c67f1d6 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -19,14 +19,12 @@ the [vinyldns organization](https://hub.docker.com/u/vinyldns/dashboard/). Namel * vinyldns/api: images for vinyldns core api engine * vinyldns/portal: images for vinyldns web client -* vinyldns/bind9: images for local DNS server used for testing The offline root key and repository keys are managed by the core maintainer team. The keys managed are: * root key: also known as the offline key, used to create the separate repository signing keys * api key: used to sign tagged images in vinyldns/api * portal key: used to sign tagged images in vinyldns/portal -* bind9 key: used to sign tagged images in the vinyldns/bind9 These keys are named in a .key format, e.g. 5526ecd15bd413e08718e66c440d17a28968d5cd2922b59a17510da802ca6572.key, do not change the names of the keys. diff --git a/README.md b/README.md index 21856b1b3..713244593 100644 --- a/README.md +++ b/README.md @@ -48,44 +48,76 @@ To start up a local instance of VinylDNS on your machine with docker: 1. Ensure that you have [docker](https://docs.docker.com/install/) and [docker-compose](https://docs.docker.com/compose/install/) 1. Clone the repo: `git clone https://github.com/vinyldns/vinyldns.git` 1. Navigate to repo: `cd vinyldns` -1. Run `./utils/quickstart-vinyldns.sh`. This will start up the api at `localhost:9000` and the portal at `localhost:9001` +1. Run `./quickstart/quickstart-vinyldns.sh`. This will start up the api at `localhost:9000` and the portal + at `localhost:9001` 1. See [Developer Guide](DEVELOPER_GUIDE.md#loading-test-data) for how to load a test DNS zone 1. To stop the local setup, run `./utils/clean-vinyldns-containers.sh`. -There exist several clients at that can be used to make API requests, using the endpoint `http://localhost:9000` +There exist several clients at that can be used to make API requests, using the +endpoint `http://localhost:9000` ## Things to try in the portal + 1. View the portal at in a web browser -1. Login with the credentials `testuser` and `testpassword` -1. Navigate to the `groups` tab: -1. Click on the **New Group** button and create a new group, the group id is the uuid in the url after you view the group -1. View zones you connected to in the `zones` tab: . For a quick test, create a new zone named "ok" with an email of "test@test.com" and choose a group you created from the previous step. (Note, see [Developer Guide](DEVELOPER_GUIDE.md#loading-test-data) for creating a zone) -1. You will see that some records are preloaded in the zoned already, this is because these records are preloaded in the local docker DNS server -and VinylDNS automatically syncs records with the backend DNS server upon zone connection -1. From here, you can create DNS record sets in the **Manage Records** tab, and manage zone settings and ***ACL rules*** -in the **Manage Zone** tab -1. To try creating a DNS record, click on the **Create Record Set** button under Records, `Record Type = A, Record Name = my-test-a, -TTL = 300, IP Addressess = 1.1.1.1` -1. Click on the **Refresh** button under Records, you should see your new record created +2. Login with the credentials `professor` and `professor` +3. Navigate to the `groups` tab: +4. Click on the **New Group** button and create a new group, the group id is the uuid in the url after you view the + group +5. View zones you connected to in the `zones` tab: . For a quick test, create a new zone + named `ok` with an email of `test@test.com` and choose a group you created from the previous step. (Note, + see [Developer Guide](DEVELOPER_GUIDE.md#loading-test-data) for creating a zone) +6. You will see that some records are preloaded in the zoned already, this is because these records are preloaded in the + local docker DNS server and VinylDNS automatically syncs records with the backend DNS server upon zone connection +7. From here, you can create DNS record sets in the **Manage Records** tab, and manage zone settings and ***ACL rules*** + in the **Manage Zone** tab +8. To try creating a DNS record, click on the **Create Record Set** button under + Records, `Record Type = A, Record Name = my-test-a, TTL = 300, IP Addressess = 1.1.1.1` +9. Click on the **Refresh** button under Records, you should see your new record created + +### Verifying Your Changes + +VinylDNS will synchronize with the DNS backend. For the Quickstart this should be running on port `19001` on `localhost`. + +To verify your changes, you can use a DNS resolution utility like `dig` + +```bash +$ dig @127.0.0.1 -p 19001 +short my-test-a.ok +1.1.1.1 +``` + +This tells `dig` to use `127.0.0.1` as the resolver on port `19001`. The `+short` just makes the output a bit less +verbose. Finally, the record we're looking up is `my-test-a.ok`. You can see the returned output of `1.1.1.1` matches +the record data we entered. ## Other things to note -1. Upon connecting to a zone for the first time, a zone sync is executed to provide VinylDNS a copy of the records in the zone -1. Changes made via VinylDNS are made against the DNS backend, you do not need to sync the zone further to push those changes out -1. If changes to the zone are made outside of VinylDNS, then the zone will have to be re-synced to give VinylDNS a copy of those records -1. If you wish to modify the url used in the creation process from `http://localhost:9000`, to say `http://vinyldns.yourdomain.com:9000`, you can modify the `utils/.env` file before execution. -1. A similar `docker/.env.quickstart` can be modified to change the default ports for the Portal and API. You must also modify their config files with the new port: https://www.vinyldns.io/operator/config-portal & https://www.vinyldns.io/operator/config-api + +1. Upon connecting to a zone for the first time, a zone sync is executed to provide VinylDNS a copy of the records in + the zone +1. Changes made via VinylDNS are made against the DNS backend, you do not need to sync the zone further to push those + changes out +1. If changes to the zone are made outside of VinylDNS, then the zone will have to be re-synced to give VinylDNS a copy + of those records +1. If you wish to modify the url used in the creation process from `http://localhost:9000`, to + say `http://vinyldns.yourdomain.com:9000`, you can modify the `quickstart/.env` file before execution. +1. Further configuration can be ac https://www.vinyldns.io/operator/config-portal + & https://www.vinyldns.io/operator/config-api ## Code of Conduct -This project, and everyone participating in it, are governed by the [VinylDNS Code Of Conduct](CODE_OF_CONDUCT.md). By -participating, you agree to this Code. Please report any violations to the code of conduct to vinyldns-core@googlegroups.com. + +This project, and everyone participating in it, are governed by the [VinylDNS Code Of Conduct](CODE_OF_CONDUCT.md). By +participating, you agree to this Code. Please report any violations to the code of conduct to +vinyldns-core@googlegroups.com. ## Developer Guide + See [DEVELOPER_GUIDE.md](DEVELOPER_GUIDE.md) for instructions on setting up VinylDNS locally. ## Contributing + See the [Contributing Guide](CONTRIBUTING.md). ## Contact + - If you have any security concerns please contact the maintainers directly vinyldns-core@googlegroups.com ## Maintainers and Contributors diff --git a/build/README.md b/build/README.md index b913d5d62..638e85e1e 100644 --- a/build/README.md +++ b/build/README.md @@ -1,80 +1,14 @@ -## Building VinylDNS +# Build -This folder contains scripts and everything you need to build and test VinylDNS from your own machine. +This folder contains scripts for building VinylDNS and it's related artifacts. -## Pre-requisites +| Path |Description | +| --- | --- | +|`assemble_api_jar.sh` | Builds the VinylDNS API jar file. You can find the resulting `jar` file in `assembly/`.| +|`deep_clean.sh` | Removes all of the build artifacts and all `target/` directories recursively.| +| `func-test-api.sh` | Runs the functional tests for the API| +| `func-test-portal.sh` | Runs the functional tests for the Portal| +| `prepare_release.sh` | Runs all of the tests in preparation for a release| +| `verify.sh` | Runs all of the unit and integration tests| -- `docker` - you will need docker and docker-compose installed locally -## Local Build and Test - -1. `./docker-release.sh --clean` -1. Open up `version.sbt` in the root to know the directory (or capture in the script output) -1. Once complete, run a test `./start.sh --version 0.9.4-SNAPSHOT` (replace 0.9.4 with the value in version.sbt). -1. Login to the portal at http://localhost:9001 to verify everything looks good -1. Run `./stop.sh` to bring everything down - -### Release Process - -1. If you are using image signing / docker notary, be sure you set the environment variable `export DOCKER_CONTENT_TRUST=1`. -Whether you sign or not is up to your organization. You need to have notary setup to be able to sign properly. -1. Be sure to login to your docker registry, typically done by `docker login` in the terminal you will release from. -1. The actual version number is pulled from the local `version.sbt` based on the branch specified (defaults to master) -1. Run `./docker-release.sh --push --clean --tag [your tag here] --branch [your branch here]` - 1. typically the `tag` is a build number that you maintain, for example a build number in Jenkins. Using this field is recommended. This value will be appended to the generated version as `-b[TAG]`; for example `0.9.4-b123` if using `123` for the tag. - 1. the `branch` defaults to `master` if not specified, you can choose any branch or tag from https://github.com/vinyldns/vinyldns -1. The version generated will be whatever the version is in the `version.sbt` on the `branch` specified (defaults to master) -1. Each of the images are built using the branch specified and the correct version -1. The func tests are run with only smoke tests against the API image to verify it is working -1. If everything passes, and the user specifies `--push`, the images are tagged and released to the docker repository (defaults to docker hub) - -### Release Script -Does a clean build off of remote master and tags it with -`./docker-release.sh --clean --push --tag 123` - -The release script is used for doing a release. It takes the following parameters: - -- `-b | --branch [BRANCH]` - what branch to pull from, can be any PR branch or a tag like `v0.9.3`, defaults to `master` -- `-c | --clean` - a flag that indicates to perform a build. If omitted, the release script will look for a -pre-built image locally -- `-p | --push` - a flag that indicates to push to the remote docker registry. The default docker registry -is `docker.io` -- `-r | --repository [REPOSITORY]` - a URL to your docker registry, defaults to `docker.io` -- `-t | --tag [TAG]` - a build qualifer for this build. For example, pass in the build number for your -continuous integration tool -- `-v | --version [VERSION]` - overrides the version calculation and forces the version passed in. Used primarily for official releases - -## Docker Images - -The build will generate several VinylDNS docker images that are used to deploy into any environment VinylDNS - -- `vinyldns/api` - this is the heart of the VinylDNS system, the backend API -- `vinyldns/portal` - the VinylDNS web UI - -### vinyldns/api - -The default build for vinyldns api assumes an **ALL MYSQL** installation. - -**Environment Variables** -- `VINYLDNS_VERSION` - this is the version of VinylDNS the API is running, typically you will not set this as -it is set as part of the container build - -**Volumes** -- `/opt/docker/conf/` - if you need to have your own application config file. This is **MANDATORY** for -any production environments. Typically, you will add your own `application.conf` file in here with your settings. -- `/opt/docker/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. -Rarely used, but if you want to bring your own message queue or database you can put the `jar` files there - -### vinyldns/portal - -The default build for vinyldns portal assumes an **ALL MYSQL** installation. - -**Environment Variables** -- `VINYLDNS_VERSION` - this is the version of VinylDNS the API is running, typically you will not set this as -it is set as part of the container build - -**Volumes** -- `/opt/docker/conf/` - if you need to have your own application config file. This is **MANDATORY** for -any production environments. Typically, you will add your own `application.conf` file in here with your settings. -- `/opt/docker/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. -Rarely used, but if you want to bring your own message queue or database you can put the `jar` files there diff --git a/build/assemble_api_jar.sh b/build/assemble_api_jar.sh new file mode 100644 index 000000000..f716e3b39 --- /dev/null +++ b/build/assemble_api_jar.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# This script will build the vinyldns.jar file using Docker. The file will +# be placed in the configured location (currently `assembly/` off of the root) +# +set -euo pipefail + +DIR=$( + cd "$(dirname "$0")" + pwd -P +) + +usage() { + echo "USAGE: assemble_jar.sh [options]" + echo -e "\t-n, --no-clean do no perform a clean before assembling the jar" + echo -e "\t-u, --update update the underlying docker image" +} + +SKIP_CLEAN=0 +UPDATE_DOCKER=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --no-clean | -n) + SKIP_CLEAN=1 + shift + ;; + --update | -u) + UPDATE_DOCKER=1 + shift + ;; + *) + usage + exit 1 + ;; + esac +done + +if ! [[ $SKIP_CLEAN -eq 1 ]]; then + "${DIR}/deep_clean.sh" +fi + +if [[ $UPDATE_DOCKER -eq 1 ]]; then + echo "Pulling latest version of 'vinyldns/build:base-test-integration'" + docker pull vinyldns/build:base-test-integration +fi + +echo "Building VinylDNS API jar file" +docker run -it --rm -e RUN_SERVICES=none -v "${DIR}/..:/build" vinyldns/build:base-test-integration -- sbt 'api/assembly' diff --git a/build/deep_clean.sh b/build/deep_clean.sh new file mode 100644 index 000000000..b30d950e4 --- /dev/null +++ b/build/deep_clean.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# This script will delete all target/ directories and the assembly/ directory +# +set -euo pipefail +DIR=$( + cd "$(dirname "$0")" + pwd -P +) + +echo "Performing deep clean" +find "${DIR}/.." -type d -name target -o -name assembly | while read -r p; do if [ -d "$p" ]; then + echo -n "Removing $p.." + rm -r "$p" || (echo -e "\e[93mError deleting $p, you may need to be root\e[0m"; exit 1) + echo "done." +fi; done diff --git a/build/docker/.env b/build/docker/.env new file mode 100644 index 000000000..f58d13e4b --- /dev/null +++ b/build/docker/.env @@ -0,0 +1,17 @@ +REST_PORT=9000 + +# portal settings +PORTAL_PORT=9001 +PLAY_HTTP_SECRET_KEY=change-this-for-prod +VINYLDNS_BACKEND_URL=http://vinyldns-integration:9000 + +SQS_ENDPOINT=http://vinyldns-integration:19003 +MYSQL_ENDPOINT=vinyldns-integration:19002 +TEST_LOGIN=true + +JDBC_DRIVER=org.mariadb.jdbc.Driver +JDBC_URL=jdbc:mariadb://vinyldns-integration:19002/vinyldns?user=root&password=pass +JDBC_MIGRATION_URL=jdbc:mariadb://vinyldns-integration:19002/?user=root&password=pass +JDBC_USER=root +JDBC_PASSWORD=pass +DEFAULT_DNS_ADDRESS=127.0.0.1:19001 diff --git a/build/docker/README.md b/build/docker/README.md new file mode 100644 index 000000000..b913d5d62 --- /dev/null +++ b/build/docker/README.md @@ -0,0 +1,80 @@ +## Building VinylDNS + +This folder contains scripts and everything you need to build and test VinylDNS from your own machine. + +## Pre-requisites + +- `docker` - you will need docker and docker-compose installed locally + +## Local Build and Test + +1. `./docker-release.sh --clean` +1. Open up `version.sbt` in the root to know the directory (or capture in the script output) +1. Once complete, run a test `./start.sh --version 0.9.4-SNAPSHOT` (replace 0.9.4 with the value in version.sbt). +1. Login to the portal at http://localhost:9001 to verify everything looks good +1. Run `./stop.sh` to bring everything down + +### Release Process + +1. If you are using image signing / docker notary, be sure you set the environment variable `export DOCKER_CONTENT_TRUST=1`. +Whether you sign or not is up to your organization. You need to have notary setup to be able to sign properly. +1. Be sure to login to your docker registry, typically done by `docker login` in the terminal you will release from. +1. The actual version number is pulled from the local `version.sbt` based on the branch specified (defaults to master) +1. Run `./docker-release.sh --push --clean --tag [your tag here] --branch [your branch here]` + 1. typically the `tag` is a build number that you maintain, for example a build number in Jenkins. Using this field is recommended. This value will be appended to the generated version as `-b[TAG]`; for example `0.9.4-b123` if using `123` for the tag. + 1. the `branch` defaults to `master` if not specified, you can choose any branch or tag from https://github.com/vinyldns/vinyldns +1. The version generated will be whatever the version is in the `version.sbt` on the `branch` specified (defaults to master) +1. Each of the images are built using the branch specified and the correct version +1. The func tests are run with only smoke tests against the API image to verify it is working +1. If everything passes, and the user specifies `--push`, the images are tagged and released to the docker repository (defaults to docker hub) + +### Release Script +Does a clean build off of remote master and tags it with +`./docker-release.sh --clean --push --tag 123` + +The release script is used for doing a release. It takes the following parameters: + +- `-b | --branch [BRANCH]` - what branch to pull from, can be any PR branch or a tag like `v0.9.3`, defaults to `master` +- `-c | --clean` - a flag that indicates to perform a build. If omitted, the release script will look for a +pre-built image locally +- `-p | --push` - a flag that indicates to push to the remote docker registry. The default docker registry +is `docker.io` +- `-r | --repository [REPOSITORY]` - a URL to your docker registry, defaults to `docker.io` +- `-t | --tag [TAG]` - a build qualifer for this build. For example, pass in the build number for your +continuous integration tool +- `-v | --version [VERSION]` - overrides the version calculation and forces the version passed in. Used primarily for official releases + +## Docker Images + +The build will generate several VinylDNS docker images that are used to deploy into any environment VinylDNS + +- `vinyldns/api` - this is the heart of the VinylDNS system, the backend API +- `vinyldns/portal` - the VinylDNS web UI + +### vinyldns/api + +The default build for vinyldns api assumes an **ALL MYSQL** installation. + +**Environment Variables** +- `VINYLDNS_VERSION` - this is the version of VinylDNS the API is running, typically you will not set this as +it is set as part of the container build + +**Volumes** +- `/opt/docker/conf/` - if you need to have your own application config file. This is **MANDATORY** for +any production environments. Typically, you will add your own `application.conf` file in here with your settings. +- `/opt/docker/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. +Rarely used, but if you want to bring your own message queue or database you can put the `jar` files there + +### vinyldns/portal + +The default build for vinyldns portal assumes an **ALL MYSQL** installation. + +**Environment Variables** +- `VINYLDNS_VERSION` - this is the version of VinylDNS the API is running, typically you will not set this as +it is set as part of the container build + +**Volumes** +- `/opt/docker/conf/` - if you need to have your own application config file. This is **MANDATORY** for +any production environments. Typically, you will add your own `application.conf` file in here with your settings. +- `/opt/docker/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. +Rarely used, but if you want to bring your own message queue or database you can put the `jar` files there diff --git a/build/docker/api/application.conf b/build/docker/api/application.conf index e3102e0d0..e4184bf97 100644 --- a/build/docker/api/application.conf +++ b/build/docker/api/application.conf @@ -152,7 +152,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.2.252", "192.0.2.253", "fd69:27cc:fe91:0:0:0:0:ffff", diff --git a/build/docker/docker-compose.yml b/build/docker/docker-compose.yml index 6c145ce8b..20d9c6438 100644 --- a/build/docker/docker-compose.yml +++ b/build/docker/docker-compose.yml @@ -1,28 +1,5 @@ version: "3.0" services: - mysql: - image: "mysql:5.7" - container_name: "vinyldns-mysql" - environment: - MYSQL_ROOT_PASSWORD: 'pass' - MYSQL_ROOT_HOST: '%' - logging: - driver: none - ports: - - "19002:3306" - - bind9: - build: - context: ./test-bind9 - args: - BRANCH: master - image: "vinyldns/test-bind9:${VINYLDNS_VERSION}" - container_name: "vinyldns-bind9" - logging: - driver: none - ports: - - "19001:53/udp" - - "19001:53" api: build: @@ -40,13 +17,28 @@ services: - ./api/application.conf:/opt/docker/conf/application.conf - ./api/logback.xml:/opt/docker/conf/logback.xml depends_on: - - mysql + - integration ldap: - image: rroemhild/test-openldap container_name: "vinyldns-ldap" + image: vinyldns/build:openldap ports: - - "19008:389" + - "19004:19004" + + integration: + container_name: "vinyldns-api-integration" + hostname: "vinyldns-integration" + image: "vinyldns-api-integration" + build: + context: ../ + dockerfile: test/api/integration/Dockerfile + environment: + RUN_SERVICES: "deps-only tail-logs" + env_file: + .env + ports: + - "19001-19003:19001-19003/tcp" + - "19001:19001/udp" portal: build: @@ -65,13 +57,3 @@ services: depends_on: - api - ldap - - functest: - build: - context: ./test - image: "vinyldns/test:${VINYLDNS_VERSION}" - environment: - TEST_PATTERN: "test_verify_production" - container_name: "vinyldns-functest" - depends_on: - - api diff --git a/build/docker-release.sh b/build/docker/docker-release.sh old mode 100755 new mode 100644 similarity index 95% rename from build/docker-release.sh rename to build/docker/docker-release.sh index 8487f90e9..51829fed6 --- a/build/docker-release.sh +++ b/build/docker/docker-release.sh @@ -95,7 +95,7 @@ export VINYLDNS_VERSION=$VINYLDNS_VERSION echo "VINYLDNS VERSION BEING RELEASED IS $VINYLDNS_VERSION" if [ $DO_BUILD -eq 1 ]; then - docker-compose -f $CURDIR/docker/docker-compose.yml build \ + docker-compose -f "${CURDIR}/docker-compose.yml" build \ --no-cache \ --parallel \ --build-arg VINYLDNS_VERSION="${VINYLDNS_VERSION}" \ @@ -103,7 +103,7 @@ if [ $DO_BUILD -eq 1 ]; then if [ $? -eq 0 ]; then # Runs smoke tests to make sure the new images are sound - docker-compose -f $CURDIR/docker/docker-compose.yml --log-level ERROR up --exit-code-from functest + docker-compose -f "${CURDIR}/docker-compose.yml" --log-level ERROR up --exit-code-from functest fi if [ $? -eq 0 ]; then diff --git a/build/start.sh b/build/docker/start.sh old mode 100755 new mode 100644 similarity index 80% rename from build/start.sh rename to build/docker/start.sh index 1b5834d4d..1f8da1c2d --- a/build/start.sh +++ b/build/docker/start.sh @@ -10,19 +10,16 @@ function usage() { } function wait_for_url() { - URL=$1 - DATA="" - RETRY="60" - echo "pinging $URL ..." + echo -n "Checking ${URL}..." + RETRY="$TIMEOUT" while [ "$RETRY" -gt 0 ]; do - DATA=$(curl -I -s "${URL}" -o /dev/null -w "%{http_code}") - if [ $? -eq 0 ]; then + if curl -I -s "${URL}" -o /dev/null -w "%{http_code}" &>/dev/null || false; then echo "Succeeded in connecting to ${URL}!" break else - echo "Retrying" >&2 + echo -n "." - let RETRY-=1 + ((RETRY -= 1)) sleep 1 if [ "$RETRY" -eq 0 ]; then @@ -58,7 +55,7 @@ else fi # Actually starts up our docker images -docker-compose -f $CURDIR/docker/docker-compose.yml up --no-build -d api portal +docker-compose -f "${CURDIR}/docker/docker-compose.yml" up --no-build -d api portal # Waits for the URL to be available wait_for_url "http://localhost:9001" @@ -68,6 +65,6 @@ if [ $? -eq 0 ]; then exit 0 else echo "VinylDNS startup failed!" - $CURDIR/stop.sh + "${CURDIR}/stop.sh" exit 1 fi diff --git a/build/stop.sh b/build/docker/stop.sh old mode 100755 new mode 100644 similarity index 100% rename from build/stop.sh rename to build/docker/stop.sh diff --git a/build/docker/test-bind9/Dockerfile b/build/docker/test-bind9/Dockerfile deleted file mode 100644 index 1bfbf4852..000000000 --- a/build/docker/test-bind9/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM alpine/git:1.0.7 as gitcheckout - -ARG BRANCH=master - -RUN git clone -b ${BRANCH} --single-branch --depth 1 https://github.com/vinyldns/vinyldns.git /vinyldns - -FROM vinyldns/bind9:0.0.5 - -COPY --from=gitcheckout /vinyldns/docker/bind9/zones/* /var/cache/bind/zones/ - -COPY --from=gitcheckout /vinyldns/docker/bind9/etc/named.conf.local /var/cache/bind/config diff --git a/utils/func-test-api.sh b/build/func-test-api.sh old mode 100755 new mode 100644 similarity index 61% rename from utils/func-test-api.sh rename to build/func-test-api.sh index 4a3050597..650e0c0c4 --- a/utils/func-test-api.sh +++ b/build/func-test-api.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# +# This script will perform the functional tests for the API using Docker +# set -euo pipefail DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) diff --git a/utils/func-test-portal.sh b/build/func-test-portal.sh old mode 100755 new mode 100644 similarity index 61% rename from utils/func-test-portal.sh rename to build/func-test-portal.sh index c40d0df60..b6534d0f1 --- a/utils/func-test-portal.sh +++ b/build/func-test-portal.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# +# This script will perform the functional tests for the Portal using Docker +# set -euo pipefail DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) diff --git a/build/prepare_release.sh b/build/prepare_release.sh new file mode 100644 index 000000000..ea9312107 --- /dev/null +++ b/build/prepare_release.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) + +source "${DIR}/../utils/includes/terminal_colors.sh" + +if [ ! -d "${DIR}/../assembly" ] || [ ! -f "${DIR}/../assembly/vinyldns.jar" ]; then + echo -e "${F_YELLOW}Warning:${F_RESET} you might want to run 'build/assemble_api_jar.sh' first to improve performance" +fi + +echo "Running unit and integration tests..." +if ! "${DIR}/verify.sh"; then + echo "Error running unit and integration tests." + exit 1 +fi + +echo "Running API functional tests..." +if ! "${DIR}/func-test-api.sh"; then + echo "Error running API functional tests" + exit 1 +fi + +echo "Running Portal functional tests..." +if ! "${DIR}/func-test-portal.sh"; then + echo "Error running Portal functional tests" + exit 1 +fi diff --git a/build/verify.sh b/build/verify.sh new file mode 100644 index 000000000..f759bb3e8 --- /dev/null +++ b/build/verify.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) + +cd "$DIR/../test/api/integration" +make build && make run WITH_ARGS="bash -c \"sbt ';validate' && sbt ';verify'\"" diff --git a/modules/api/src/it/resources/application.conf b/modules/api/src/it/resources/application.conf index f792c5d76..df6975a45 100644 --- a/modules/api/src/it/resources/application.conf +++ b/modules/api/src/it/resources/application.conf @@ -66,7 +66,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.2.252", "192.0.2.253", "fd69:27cc:fe91:0:0:0:0:ffff", diff --git a/modules/api/src/main/resources/application.conf b/modules/api/src/main/resources/application.conf index 2808e868d..433095214 100644 --- a/modules/api/src/main/resources/application.conf +++ b/modules/api/src/main/resources/application.conf @@ -41,7 +41,7 @@ vinyldns { access-key = "x" secret-key = "x" } - } + } data-stores = ["mysql"] mysql { @@ -105,7 +105,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.2.252", "192.0.2.253", "fd69:27cc:fe91:0:0:0:0:ffff", diff --git a/test/api/functional/test/.gitignore b/modules/api/src/test/functional/.gitignore old mode 100755 new mode 100644 similarity index 100% rename from test/api/functional/test/.gitignore rename to modules/api/src/test/functional/.gitignore diff --git a/test/api/functional/test/__init__.py b/modules/api/src/test/functional/__init__.py similarity index 100% rename from test/api/functional/test/__init__.py rename to modules/api/src/test/functional/__init__.py diff --git a/test/api/functional/test/aws_request_signer.py b/modules/api/src/test/functional/aws_request_signer.py similarity index 100% rename from test/api/functional/test/aws_request_signer.py rename to modules/api/src/test/functional/aws_request_signer.py diff --git a/test/api/functional/test/conftest.py b/modules/api/src/test/functional/conftest.py similarity index 100% rename from test/api/functional/test/conftest.py rename to modules/api/src/test/functional/conftest.py diff --git a/test/api/functional/test/pytest.ini b/modules/api/src/test/functional/pytest.ini similarity index 100% rename from test/api/functional/test/pytest.ini rename to modules/api/src/test/functional/pytest.ini diff --git a/test/api/functional/test/pytest.sh b/modules/api/src/test/functional/pytest.sh old mode 100755 new mode 100644 similarity index 100% rename from test/api/functional/test/pytest.sh rename to modules/api/src/test/functional/pytest.sh diff --git a/test/api/functional/test/requirements.txt b/modules/api/src/test/functional/requirements.txt similarity index 100% rename from test/api/functional/test/requirements.txt rename to modules/api/src/test/functional/requirements.txt diff --git a/test/api/functional/test/run.sh b/modules/api/src/test/functional/run.sh old mode 100755 new mode 100644 similarity index 100% rename from test/api/functional/test/run.sh rename to modules/api/src/test/functional/run.sh diff --git a/test/api/functional/test/tests/authentication_test.py b/modules/api/src/test/functional/tests/authentication_test.py similarity index 100% rename from test/api/functional/test/tests/authentication_test.py rename to modules/api/src/test/functional/tests/authentication_test.py diff --git a/test/api/functional/test/tests/batch/approve_batch_change_test.py b/modules/api/src/test/functional/tests/batch/approve_batch_change_test.py similarity index 100% rename from test/api/functional/test/tests/batch/approve_batch_change_test.py rename to modules/api/src/test/functional/tests/batch/approve_batch_change_test.py diff --git a/test/api/functional/test/tests/batch/cancel_batch_change_test.py b/modules/api/src/test/functional/tests/batch/cancel_batch_change_test.py similarity index 100% rename from test/api/functional/test/tests/batch/cancel_batch_change_test.py rename to modules/api/src/test/functional/tests/batch/cancel_batch_change_test.py diff --git a/test/api/functional/test/tests/batch/create_batch_change_test.py b/modules/api/src/test/functional/tests/batch/create_batch_change_test.py similarity index 100% rename from test/api/functional/test/tests/batch/create_batch_change_test.py rename to modules/api/src/test/functional/tests/batch/create_batch_change_test.py diff --git a/test/api/functional/test/tests/batch/get_batch_change_test.py b/modules/api/src/test/functional/tests/batch/get_batch_change_test.py similarity index 100% rename from test/api/functional/test/tests/batch/get_batch_change_test.py rename to modules/api/src/test/functional/tests/batch/get_batch_change_test.py diff --git a/test/api/functional/test/tests/batch/list_batch_change_summaries_test.py b/modules/api/src/test/functional/tests/batch/list_batch_change_summaries_test.py similarity index 100% rename from test/api/functional/test/tests/batch/list_batch_change_summaries_test.py rename to modules/api/src/test/functional/tests/batch/list_batch_change_summaries_test.py diff --git a/test/api/functional/test/tests/batch/reject_batch_change_test.py b/modules/api/src/test/functional/tests/batch/reject_batch_change_test.py similarity index 100% rename from test/api/functional/test/tests/batch/reject_batch_change_test.py rename to modules/api/src/test/functional/tests/batch/reject_batch_change_test.py diff --git a/test/api/functional/test/tests/conftest.py b/modules/api/src/test/functional/tests/conftest.py similarity index 100% rename from test/api/functional/test/tests/conftest.py rename to modules/api/src/test/functional/tests/conftest.py diff --git a/test/api/functional/test/tests/internal/color_test.py b/modules/api/src/test/functional/tests/internal/color_test.py similarity index 100% rename from test/api/functional/test/tests/internal/color_test.py rename to modules/api/src/test/functional/tests/internal/color_test.py diff --git a/test/api/functional/test/tests/internal/health_test.py b/modules/api/src/test/functional/tests/internal/health_test.py similarity index 100% rename from test/api/functional/test/tests/internal/health_test.py rename to modules/api/src/test/functional/tests/internal/health_test.py diff --git a/test/api/functional/test/tests/internal/ping_test.py b/modules/api/src/test/functional/tests/internal/ping_test.py similarity index 100% rename from test/api/functional/test/tests/internal/ping_test.py rename to modules/api/src/test/functional/tests/internal/ping_test.py diff --git a/test/api/functional/test/tests/internal/status_test.py b/modules/api/src/test/functional/tests/internal/status_test.py similarity index 100% rename from test/api/functional/test/tests/internal/status_test.py rename to modules/api/src/test/functional/tests/internal/status_test.py diff --git a/test/api/functional/test/tests/list_batch_summaries_test_context.py b/modules/api/src/test/functional/tests/list_batch_summaries_test_context.py similarity index 100% rename from test/api/functional/test/tests/list_batch_summaries_test_context.py rename to modules/api/src/test/functional/tests/list_batch_summaries_test_context.py diff --git a/test/api/functional/test/tests/list_groups_test_context.py b/modules/api/src/test/functional/tests/list_groups_test_context.py similarity index 100% rename from test/api/functional/test/tests/list_groups_test_context.py rename to modules/api/src/test/functional/tests/list_groups_test_context.py diff --git a/test/api/functional/test/tests/list_recordsets_test_context.py b/modules/api/src/test/functional/tests/list_recordsets_test_context.py similarity index 100% rename from test/api/functional/test/tests/list_recordsets_test_context.py rename to modules/api/src/test/functional/tests/list_recordsets_test_context.py diff --git a/test/api/functional/test/tests/list_zones_test_context.py b/modules/api/src/test/functional/tests/list_zones_test_context.py similarity index 100% rename from test/api/functional/test/tests/list_zones_test_context.py rename to modules/api/src/test/functional/tests/list_zones_test_context.py diff --git a/test/api/functional/test/tests/membership/create_group_test.py b/modules/api/src/test/functional/tests/membership/create_group_test.py similarity index 100% rename from test/api/functional/test/tests/membership/create_group_test.py rename to modules/api/src/test/functional/tests/membership/create_group_test.py diff --git a/test/api/functional/test/tests/membership/delete_group_test.py b/modules/api/src/test/functional/tests/membership/delete_group_test.py similarity index 100% rename from test/api/functional/test/tests/membership/delete_group_test.py rename to modules/api/src/test/functional/tests/membership/delete_group_test.py diff --git a/test/api/functional/test/tests/membership/get_group_changes_test.py b/modules/api/src/test/functional/tests/membership/get_group_changes_test.py similarity index 100% rename from test/api/functional/test/tests/membership/get_group_changes_test.py rename to modules/api/src/test/functional/tests/membership/get_group_changes_test.py diff --git a/test/api/functional/test/tests/membership/get_group_test.py b/modules/api/src/test/functional/tests/membership/get_group_test.py similarity index 100% rename from test/api/functional/test/tests/membership/get_group_test.py rename to modules/api/src/test/functional/tests/membership/get_group_test.py diff --git a/test/api/functional/test/tests/membership/list_group_admins_test.py b/modules/api/src/test/functional/tests/membership/list_group_admins_test.py similarity index 100% rename from test/api/functional/test/tests/membership/list_group_admins_test.py rename to modules/api/src/test/functional/tests/membership/list_group_admins_test.py diff --git a/test/api/functional/test/tests/membership/list_group_members_test.py b/modules/api/src/test/functional/tests/membership/list_group_members_test.py similarity index 100% rename from test/api/functional/test/tests/membership/list_group_members_test.py rename to modules/api/src/test/functional/tests/membership/list_group_members_test.py diff --git a/test/api/functional/test/tests/membership/list_my_groups_test.py b/modules/api/src/test/functional/tests/membership/list_my_groups_test.py similarity index 100% rename from test/api/functional/test/tests/membership/list_my_groups_test.py rename to modules/api/src/test/functional/tests/membership/list_my_groups_test.py diff --git a/test/api/functional/test/tests/membership/update_group_test.py b/modules/api/src/test/functional/tests/membership/update_group_test.py similarity index 100% rename from test/api/functional/test/tests/membership/update_group_test.py rename to modules/api/src/test/functional/tests/membership/update_group_test.py diff --git a/test/api/functional/test/tests/production_verify_test.py b/modules/api/src/test/functional/tests/production_verify_test.py similarity index 100% rename from test/api/functional/test/tests/production_verify_test.py rename to modules/api/src/test/functional/tests/production_verify_test.py diff --git a/test/api/functional/test/tests/recordsets/create_recordset_test.py b/modules/api/src/test/functional/tests/recordsets/create_recordset_test.py similarity index 100% rename from test/api/functional/test/tests/recordsets/create_recordset_test.py rename to modules/api/src/test/functional/tests/recordsets/create_recordset_test.py diff --git a/test/api/functional/test/tests/recordsets/delete_recordset_test.py b/modules/api/src/test/functional/tests/recordsets/delete_recordset_test.py similarity index 100% rename from test/api/functional/test/tests/recordsets/delete_recordset_test.py rename to modules/api/src/test/functional/tests/recordsets/delete_recordset_test.py diff --git a/test/api/functional/test/tests/recordsets/get_recordset_test.py b/modules/api/src/test/functional/tests/recordsets/get_recordset_test.py similarity index 100% rename from test/api/functional/test/tests/recordsets/get_recordset_test.py rename to modules/api/src/test/functional/tests/recordsets/get_recordset_test.py diff --git a/test/api/functional/test/tests/recordsets/list_recordset_changes_test.py b/modules/api/src/test/functional/tests/recordsets/list_recordset_changes_test.py similarity index 100% rename from test/api/functional/test/tests/recordsets/list_recordset_changes_test.py rename to modules/api/src/test/functional/tests/recordsets/list_recordset_changes_test.py diff --git a/test/api/functional/test/tests/recordsets/list_recordsets_test.py b/modules/api/src/test/functional/tests/recordsets/list_recordsets_test.py similarity index 100% rename from test/api/functional/test/tests/recordsets/list_recordsets_test.py rename to modules/api/src/test/functional/tests/recordsets/list_recordsets_test.py diff --git a/test/api/functional/test/tests/recordsets/update_recordset_test.py b/modules/api/src/test/functional/tests/recordsets/update_recordset_test.py similarity index 100% rename from test/api/functional/test/tests/recordsets/update_recordset_test.py rename to modules/api/src/test/functional/tests/recordsets/update_recordset_test.py diff --git a/test/api/functional/test/tests/shared_zone_test_context.py b/modules/api/src/test/functional/tests/shared_zone_test_context.py similarity index 100% rename from test/api/functional/test/tests/shared_zone_test_context.py rename to modules/api/src/test/functional/tests/shared_zone_test_context.py diff --git a/test/api/functional/test/tests/test_data.py b/modules/api/src/test/functional/tests/test_data.py similarity index 100% rename from test/api/functional/test/tests/test_data.py rename to modules/api/src/test/functional/tests/test_data.py diff --git a/test/api/functional/test/tests/zones/create_zone_test.py b/modules/api/src/test/functional/tests/zones/create_zone_test.py similarity index 100% rename from test/api/functional/test/tests/zones/create_zone_test.py rename to modules/api/src/test/functional/tests/zones/create_zone_test.py diff --git a/test/api/functional/test/tests/zones/delete_zone_test.py b/modules/api/src/test/functional/tests/zones/delete_zone_test.py similarity index 100% rename from test/api/functional/test/tests/zones/delete_zone_test.py rename to modules/api/src/test/functional/tests/zones/delete_zone_test.py diff --git a/test/api/functional/test/tests/zones/get_zone_test.py b/modules/api/src/test/functional/tests/zones/get_zone_test.py similarity index 100% rename from test/api/functional/test/tests/zones/get_zone_test.py rename to modules/api/src/test/functional/tests/zones/get_zone_test.py diff --git a/test/api/functional/test/tests/zones/list_zone_changes_test.py b/modules/api/src/test/functional/tests/zones/list_zone_changes_test.py similarity index 100% rename from test/api/functional/test/tests/zones/list_zone_changes_test.py rename to modules/api/src/test/functional/tests/zones/list_zone_changes_test.py diff --git a/test/api/functional/test/tests/zones/list_zones_test.py b/modules/api/src/test/functional/tests/zones/list_zones_test.py similarity index 100% rename from test/api/functional/test/tests/zones/list_zones_test.py rename to modules/api/src/test/functional/tests/zones/list_zones_test.py diff --git a/test/api/functional/test/tests/zones/sync_zone_test.py b/modules/api/src/test/functional/tests/zones/sync_zone_test.py similarity index 100% rename from test/api/functional/test/tests/zones/sync_zone_test.py rename to modules/api/src/test/functional/tests/zones/sync_zone_test.py diff --git a/test/api/functional/test/tests/zones/update_zone_test.py b/modules/api/src/test/functional/tests/zones/update_zone_test.py similarity index 100% rename from test/api/functional/test/tests/zones/update_zone_test.py rename to modules/api/src/test/functional/tests/zones/update_zone_test.py diff --git a/test/api/functional/test/utils.py b/modules/api/src/test/functional/utils.py similarity index 100% rename from test/api/functional/test/utils.py rename to modules/api/src/test/functional/utils.py diff --git a/test/api/functional/test/vinyldns_context.py b/modules/api/src/test/functional/vinyldns_context.py similarity index 100% rename from test/api/functional/test/vinyldns_context.py rename to modules/api/src/test/functional/vinyldns_context.py diff --git a/test/api/functional/test/vinyldns_python.py b/modules/api/src/test/functional/vinyldns_python.py similarity index 100% rename from test/api/functional/test/vinyldns_python.py rename to modules/api/src/test/functional/vinyldns_python.py diff --git a/modules/api/src/test/resources/application.conf b/modules/api/src/test/resources/application.conf index 073c385e5..b1bd68c8d 100644 --- a/modules/api/src/test/resources/application.conf +++ b/modules/api/src/test/resources/application.conf @@ -33,7 +33,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.2.252", "192.0.2.253", "fd69:27cc:fe91:0:0:0:0:ffff", diff --git a/modules/api/src/test/scala/vinyldns/api/ResultHelpers.scala b/modules/api/src/test/scala/vinyldns/api/ResultHelpers.scala index 79db6ac23..2fe473fb8 100644 --- a/modules/api/src/test/scala/vinyldns/api/ResultHelpers.scala +++ b/modules/api/src/test/scala/vinyldns/api/ResultHelpers.scala @@ -35,13 +35,13 @@ trait ResultHelpers { private implicit val cs: ContextShift[IO] = IO.contextShift(scala.concurrent.ExecutionContext.global) - def await[T](f: => IO[_], duration: FiniteDuration = 1.second): T = + def await[T](f: => IO[_], duration: FiniteDuration = 60.seconds): T = awaitResultOf[T](f.map(_.asInstanceOf[T]).attempt, duration).toOption.get // Waits for the future to complete, then returns the value as an Either[Throwable, T] def awaitResultOf[T]( f: => IO[Either[Throwable, T]], - duration: FiniteDuration = 1.second + duration: FiniteDuration = 60.seconds ): Either[Throwable, T] = { val timeOut = IO.sleep(duration) *> IO( @@ -55,7 +55,7 @@ trait ResultHelpers { } // Assumes that the result of the future operation will be successful, this will fail on a left disjunction - def rightResultOf[T](f: => IO[Either[Throwable, T]], duration: FiniteDuration = 1.second): T = + def rightResultOf[T](f: => IO[Either[Throwable, T]], duration: FiniteDuration = 60.seconds): T = awaitResultOf[T](f, duration) match { case Right(result) => result case Left(error) => throw error @@ -64,7 +64,7 @@ trait ResultHelpers { // Assumes that the result of the future operation will fail, this will error on a right disjunction def leftResultOf[T]( f: => IO[Either[Throwable, T]], - duration: FiniteDuration = 1.second + duration: FiniteDuration = 60.seconds ): Throwable = awaitResultOf(f, duration).swap.toOption.get def leftValue[T](t: Either[Throwable, T]): Throwable = t.swap.toOption.get diff --git a/modules/api/src/universal/conf/application.conf b/modules/api/src/universal/conf/application.conf index 9b9cf7b69..6b90dbe92 100644 --- a/modules/api/src/universal/conf/application.conf +++ b/modules/api/src/universal/conf/application.conf @@ -79,7 +79,7 @@ vinyldns { name = "vinyldns." keyName = "vinyldns." key = "nzisn+4G2ldMn0q1CV3vsg==" - primaryServer = "vinyldns-bind9" + primaryServer = "localhost:19001" } # the AXFR connection information for the default dns backend @@ -87,7 +87,7 @@ vinyldns { name = "vinyldns." keyName = "vinyldns." key = "nzisn+4G2ldMn0q1CV3vsg==" - primaryServer = "vinyldns-bind9" + primaryServer = "localhost:19001" } backends = [ @@ -97,13 +97,13 @@ vinyldns { name = "vinyldns." key-name = "vinyldns." key = "nzisn+4G2ldMn0q1CV3vsg==" - primary-server = "vinyldns-bind9" + primary-server = "localhost:19001" } transfer-connection { name = "vinyldns." key-name = "vinyldns." key = "nzisn+4G2ldMn0q1CV3vsg==" - primary-server = "vinyldns-bind9" + primary-server = "localhost:19001" } } ] diff --git a/modules/docs/src/main/mdoc/operator/config-api.md b/modules/docs/src/main/mdoc/operator/config-api.md index 521d98db9..d07a6821d 100644 --- a/modules/docs/src/main/mdoc/operator/config-api.md +++ b/modules/docs/src/main/mdoc/operator/config-api.md @@ -303,7 +303,7 @@ vinyldns { name = "vinyldns." keyName = "vinyldns." key = "nzisn+4G2ldMn0q1CV3vsg==" - primaryServer = "vinyldns-bind9" + primaryServer = "localhost:19001" algorithm = "HMAC-MD5" } } @@ -609,7 +609,7 @@ vinyldns { name = "vinyldns." keyName = "vinyldns." key = "nzisn+4G2ldMn0q1CV3vsg==" - primaryServer = "vinyldns-bind9" + primaryServer = "localhost:19001" algorithm = "HMAC-MD5" } @@ -618,7 +618,7 @@ vinyldns { name = "vinyldns." keyName = "vinyldns." key = "nzisn+4G2ldMn0q1CV3vsg==" - primaryServer = "vinyldns-bind9" + primaryServer = "localhost:19001" algorithm = "HMAC-MD5" } diff --git a/quickstart/.env b/quickstart/.env index f58d13e4b..a5d9143db 100644 --- a/quickstart/.env +++ b/quickstart/.env @@ -1,17 +1,22 @@ -REST_PORT=9000 +# General settings +VINYLDNS_API_URL=http://localhost:9000 +VINYLDNS_PORTAL_URL=http://localhost:9001 -# portal settings +# Portal settings PORTAL_PORT=9001 PLAY_HTTP_SECRET_KEY=change-this-for-prod -VINYLDNS_BACKEND_URL=http://vinyldns-integration:9000 +VINYLDNS_BACKEND_URL=http://vinyldns-api:9000 +TEST_LOGIN=false +# API Settings +REST_PORT=9000 SQS_ENDPOINT=http://vinyldns-integration:19003 +SNS_SERVICE_ENDPOINT=http://vinyldns-integration:19003 MYSQL_ENDPOINT=vinyldns-integration:19002 -TEST_LOGIN=true +DEFAULT_DNS_ADDRESS=vinyldns-integration:19001 JDBC_DRIVER=org.mariadb.jdbc.Driver JDBC_URL=jdbc:mariadb://vinyldns-integration:19002/vinyldns?user=root&password=pass JDBC_MIGRATION_URL=jdbc:mariadb://vinyldns-integration:19002/?user=root&password=pass JDBC_USER=root JDBC_PASSWORD=pass -DEFAULT_DNS_ADDRESS=127.0.0.1:19001 diff --git a/quickstart/README.md b/quickstart/README.md new file mode 100644 index 000000000..3128d7f78 --- /dev/null +++ b/quickstart/README.md @@ -0,0 +1,35 @@ +# VinylDNS QuickStart + +To get started with VinylDNS, you can experiment with the QuickStart. + +QuickStart utilizes Docker Compose to start up the VinylDNS API and Portal along with required dependencies such as: + +- MySQL +- OpenLDAP +- SQS +- SNS +- BIND 9 + +## Running + +To run the QuickStart, you will need the following prerequisites: + +- Docker +- Docker Compose + +From a shell in the `quickstart/` directory, simply run: + +```shell script +./quickstart-vinyldns.sh +``` +The `quickstart-vinyldns.sh` script takes a number of optional arguments: + +| Flag | Description | +|:---|:---| +| -a, --api-only | do not start up the VinylDNS Portal| +| -b, --build | force a rebuild of the Docker images with the local code| +| -r, --reset | reset any the running containers| +| -s, --service | specify the service to run| +| -t, --timeout | the time to wait (in seconds) for the portal and API to start (default: 60)| +| -u, --update | remove the local quickstart images to force a re-pull from docker hub| +| -v, --version-tag | specify Docker image tag version (default: latest)| diff --git a/quickstart/api/Dockerfile b/quickstart/api/Dockerfile new file mode 100644 index 000000000..5bd5c9696 --- /dev/null +++ b/quickstart/api/Dockerfile @@ -0,0 +1,39 @@ +# Build VinylDNS API if the JAR doesn't already exist +ARG VINYLDNS_VERSION=latest +FROM vinyldns/build:base-build as base-build +ARG DOCKER_FILE_PATH +COPY . /build/ +WORKDIR /build + +## Run the build if we don't already have a vinyldns.jar +RUN mkdir -p /opt/vinyldns/conf && \ + if [ -f assembly/vinyldns.jar ]; then cp assembly/vinyldns.jar /opt/vinyldns/; fi && \ + if [ ! -f /opt/vinyldns/vinyldns.jar ]; then \ + env SBT_OPTS="-XX:+UseConcMarkSweepGC -Xmx4G -Xms1G" \ + sbt -Dbuild.scalafmtOnCompile=false -Dbuild.lintOnCompile=fase ";project api;coverageOff;assembly" \ + && cp assembly/vinyldns.jar /opt/vinyldns/; \ + fi + +FROM adoptopenjdk/openjdk11:jdk-11.0.8_10-alpine + +RUN apk add --update --no-cache bash + +COPY --from=base-build /opt/vinyldns /opt/vinyldns + +# This will set the vinyldns version, make sure to have this in config... version = ${?VINYLDNS_VERSION} +ARG VINYLDNS_VERSION="0.0.0-local-dev" +ENV VINYLDNS_VERSION=$VINYLDNS_VERSION + +RUN mkdir -p /opt/vinyldns/lib_extra + +# Mount the volume for config file and lib extras +# Note: These volume names are used in the build.sbt +VOLUME ["/opt/vinyldns/lib_extra/", "/opt/vinyldns/conf"] + +EXPOSE 9000 + +ENTRYPOINT ["java", "-Dconfig.file=/opt/vinyldns/conf/vinyldns.conf", \ + "-Dlogback.configurationFile=/opt/vinyldns/conf/logback.xml",\ + "-Dvinyldns.base-version=${VINYLDNS_VERSION}",\ + "-cp", "/opt/vinyldns/lib_extra/*",\ + "-jar", "/opt/vinyldns/vinyldns.jar"] diff --git a/quickstart/api/Makefile b/quickstart/api/Makefile new file mode 100644 index 000000000..ba8249efe --- /dev/null +++ b/quickstart/api/Makefile @@ -0,0 +1,46 @@ +SHELL=bash +IMAGE_NAME=vinyldns/api:local-dev +ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) + +# Check that the required version of make is being used +REQ_MAKE_VER:=3.82 +ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) + $(error The version of MAKE $(REQ_MAKE_VER) or higher is required; you are running $(MAKE_VERSION)) +endif + +# Extract arguments for `make run` +EXTRACT_ARGS=true +ifeq (run,$(firstword $(MAKECMDGOALS))) + EXTRACT_ARGS=true +endif +ifeq ($(EXTRACT_ARGS),true) + # use the rest as arguments for "run" + WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) +endif +ifdef $(WITH_ARGS) + ARG_SEPARATOR=-- +endif + +%: + @: + +.ONESHELL: + +.PHONY: all build run + +all: build run + +build: + @set -euo pipefail + cd ../.. + docker build -t $(IMAGE_NAME) --build-arg DOCKER_FILE_PATH="$$(realpath --relative-to="." "$(ROOT_DIR)")" -f "$(ROOT_DIR)/Dockerfile" . + +run: + @set -euo pipefail + docker run -it --rm $(DOCKER_PARAMS) -v "$$(pwd)/application.conf:/opt/vinyldns/conf/vinyldns.conf" -v "$$(pwd)/logback.xml:/opt/vinyldns/conf/logback.xml" -p 9000:9000 $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) + +run-bg: + @set -euo pipefail + docker stop $(IMAGE_NAME) &> /dev/null || true + docker rm $(IMAGE_NAME) &> /dev/null || true + docker run -td --name $(IMAGE_NAME) --rm $(DOCKER_PARAMS) -v "$$(pwd)/application.conf:/opt/vinyldns/conf/vinyldns.conf" -v "$$(pwd)/logback.xml:/opt/vinyldns/conf/logback.xml" -p 9001:9001 $(IMAGE_NAME) -- /bin/bash diff --git a/quickstart/api/application.conf b/quickstart/api/application.conf new file mode 100644 index 000000000..58903a17e --- /dev/null +++ b/quickstart/api/application.conf @@ -0,0 +1,346 @@ +vinyldns { + + base-version = "0.0.0-local-dev" + version = ${vinyldns.base-version} # default to the base version if not overridden + version = ${?VINYLDNS_VERSION} # override the base version via env var + + # How often to any particular zone can be synchronized in milliseconds + sync-delay = 10000 + sync-delay = ${?SYNC_DELAY} + + # If we should start up polling for change requests, set this to false for the inactive cluster + processing-disabled = false + processing-disabled = ${?PROCESSING_DISABLED} + + # Number of records that can be in a zone + max-zone-size = 60000 + max-zone-size = ${?MAX_ZONE_SIZE} + + # Types of unowned records that users can access in shared zones + shared-approved-types = ["A", "AAAA", "CNAME", "PTR", "TXT"] + + # Batch change settings + batch-change-limit = 1000 + batch-change-limit = ${?BATCH_CHANGE_LIMIT} + manual-batch-review-enabled = true + manual-batch-review-enabled = ${?MANUAL_BATCH_REVIEW_ENABLED} + scheduled-changes-enabled = true + scheduled-changes-enabled = ${?SCHEDULED_CHANGES_ENABLED} + multi-record-batch-change-enabled = true + multi-record-batch-change-enabled = ${?MULTI_RECORD_BATCH_CHANGE_ENABLED} + + # configured backend providers + backend { + # Use "default" when dns backend legacy = true + # otherwise, use the id of one of the connections in any of your backends + default-backend-id = "default" + + # this is where we can save additional backends + backend-providers = [ + { + class-name = "vinyldns.api.backend.dns.DnsBackendProviderLoader" + settings = { + legacy = false + backends = [ + { + id = "default" + zone-connection = { + name = "vinyldns." + key-name = "vinyldns." + key-name = ${?DEFAULT_DNS_KEY_NAME} + key = "nzisn+4G2ldMn0q1CV3vsg==" + key = ${?DEFAULT_DNS_KEY_SECRET} + primary-server = "127.0.0.1" + primary-server = ${?DEFAULT_DNS_ADDRESS} + } + transfer-connection = { + name = "vinyldns." + key-name = "vinyldns." + key-name = ${?DEFAULT_DNS_KEY_NAME} + key = "nzisn+4G2ldMn0q1CV3vsg==" + key = ${?DEFAULT_DNS_KEY_SECRET} + primary-server = "127.0.0.1" + primary-server = ${?DEFAULT_DNS_ADDRESS} + }, + tsig-usage = "always" + }, + { + id = "func-test-backend" + zone-connection = { + name = "vinyldns." + key-name = "vinyldns." + key-name = ${?DEFAULT_DNS_KEY_NAME} + key = "nzisn+4G2ldMn0q1CV3vsg==" + key = ${?DEFAULT_DNS_KEY_SECRET} + primary-server = "127.0.0.1" + primary-server = ${?DEFAULT_DNS_ADDRESS} + } + transfer-connection = { + name = "vinyldns." + key-name = "vinyldns." + key-name = ${?DEFAULT_DNS_KEY_NAME} + key = "nzisn+4G2ldMn0q1CV3vsg==" + key = ${?DEFAULT_DNS_KEY_SECRET} + primary-server = "127.0.0.1" + primary-server = ${?DEFAULT_DNS_ADDRESS} + }, + tsig-usage = "always" + } + ] + } + } + ] + } + + + queue { + class-name = "vinyldns.sqs.queue.SqsMessageQueueProvider" + + messages-per-poll = 10 + polling-interval = 250.millis + + settings { + # AWS access key and secret. + access-key = "test" + access-key = ${?AWS_ACCESS_KEY} + secret-key = "test" + secret-key = ${?AWS_SECRET_ACCESS_KEY} + + # Regional endpoint to make your requests (eg. 'us-west-2', 'us-east-1', etc.). This is the region where your queue is housed. + signing-region = "us-east-1" + signing-region = ${?SQS_REGION} + + # Endpoint to access queue + service-endpoint = "http://vinyldns-integration:19003/" + service-endpoint = ${?SQS_SERVICE_ENDPOINT} + + # Queue name. Should be used in conjunction with service endpoint, rather than using a queue url which is subject to change. + queue-name = "vinyldns" + queue-name = ${?SQS_QUEUE_NAME} + } + } + + email { + class-name = "vinyldns.api.notifier.email.EmailNotifierProvider" + class-name = ${?EMAIL_CLASS_NAME} + settings = { + from = "VinylDNS " + } + } + + sns { + class-name = "vinyldns.apadi.notifier.sns.SnsNotifierProvider" + class-name = ${?SNS_CLASS_NAME} + settings { + topic-arn = "arn:aws:sns:us-east-1:000000000000:batchChanges" + topic-arn = ${?SNS_TOPIC_ARN} + access-key = "test" + access-key = ${?SNS_ACCESS_KEY} + secret-key = "test" + secret-key = ${?SNS_SECRET_KEY} + service-endpoint = "http://vinyldns-integration:19003" + service-endpoint = ${?SNS_SERVICE_ENDPOINT} + signing-region = "us-east-1" + signing-region = ${?SNS_REGION} + } + } + + rest { + host = "0.0.0.0" + port = 9000 + port=${?API_SERVICE_PORT} + } + + + approved-name-servers = [ + "172.17.42.1.", + "ns1.parent.com." + "ns1.parent.com1." + "ns1.parent.com2." + "ns1.parent.com3." + "ns1.parent.com4." + ] + + crypto { + type = "vinyldns.core.crypto.NoOpCrypto" + } + + data-stores = ["mysql"] + + mysql { + settings { + # JDBC Settings, these are all values in scalikejdbc-config, not our own + # these must be overridden to use MYSQL for production use + # assumes a docker or mysql instance running locally + name = "vinyldns" + name = ${?DATABASE_NAME} + driver = "org.h2.Driver" + driver = ${?JDBC_DRIVER} + migration-url = "jdbc:h2:mem:vinyldns;MODE=MYSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE;IGNORECASE=TRUE;INIT=RUNSCRIPT FROM 'classpath:test/ddl.sql'" + migration-url = ${?JDBC_MIGRATION_URL} + url = "jdbc:h2:mem:vinyldns;MODE=MYSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE;IGNORECASE=TRUE;INIT=RUNSCRIPT FROM 'classpath:test/ddl.sql'" + url = ${?JDBC_URL} + user = "sa" + user = ${?JDBC_USER} + password = "" + password = ${?JDBC_PASSWORD} + # see https://github.com/brettwooldridge/HikariCP + connection-timeout-millis = 1000 + idle-timeout = 10000 + max-lifetime = 600000 + maximum-pool-size = 20 + minimum-idle = 20 + register-mbeans = true + } + # Repositories that use this data store are listed here + repositories { + zone { + # no additional settings for now + } + batch-change { + # no additional settings for now + } + user { + + } + record-set { + + } + group { + + } + membership { + + } + group-change { + + } + zone-change { + + } + record-change { + + } + } + } + + backends = [] + + + # FQDNs / IPs that cannot be modified via VinylDNS + # regex-list used for all record types except PTR + # ip-list used exclusively for PTR records + high-value-domains = { + regex-list = [ + "high-value-domain.*" # for testing + ] + ip-list = [ + # using reverse zones in the vinyldns/bind9 docker image for testing + "192.0.2.252", + "192.0.2.253", + "fd69:27cc:fe91:0:0:0:0:ffff", + "fd69:27cc:fe91:0:0:0:ffff:0" + ] + } + + # FQDNs / IPs / zone names that require manual review upon submission in batch change interface + # domain-list used for all record types except PTR + # ip-list used exclusively for PTR records + manual-review-domains = { + domain-list = [ + "needs-review.*" + ] + ip-list = [ + "192.0.1.254", + "192.0.1.255", + "192.0.2.254", + "192.0.2.255", + "192.0.3.254", + "192.0.3.255", + "192.0.4.254", + "192.0.4.255", + "fd69:27cc:fe91:0:0:0:ffff:1", + "fd69:27cc:fe91:0:0:0:ffff:2", + "fd69:27cc:fe92:0:0:0:ffff:1", + "fd69:27cc:fe92:0:0:0:ffff:2", + "fd69:27cc:fe93:0:0:0:ffff:1", + "fd69:27cc:fe93:0:0:0:ffff:2", + "fd69:27cc:fe94:0:0:0:ffff:1", + "fd69:27cc:fe94:0:0:0:ffff:2" + ] + zone-name-list = [ + "zone.requires.review." + "zone.requires.review1." + "zone.requires.review2." + "zone.requires.review3." + "zone.requires.review4." + ] + } + + # FQDNs / IPs that cannot be modified via VinylDNS + # regex-list used for all record types except PTR + # ip-list used exclusively for PTR records + high-value-domains = { + regex-list = [ + "high-value-domain.*" # for testing + ] + ip-list = [ + # using reverse zones in the vinyldns/bind9 docker image for testing + "192.0.1.252", + "192.0.1.253", + "192.0.2.252", + "192.0.2.253", + "192.0.3.252", + "192.0.3.253", + "192.0.4.252", + "192.0.4.253", + "fd69:27cc:fe91:0:0:0:0:ffff", + "fd69:27cc:fe91:0:0:0:ffff:0", + "fd69:27cc:fe92:0:0:0:0:ffff", + "fd69:27cc:fe92:0:0:0:ffff:0", + "fd69:27cc:fe93:0:0:0:0:ffff", + "fd69:27cc:fe93:0:0:0:ffff:0", + "fd69:27cc:fe94:0:0:0:0:ffff", + "fd69:27cc:fe94:0:0:0:ffff:0" + ] + } + + global-acl-rules = [ + { + group-ids: ["global-acl-group-id"], + fqdn-regex-list: [".*shared[0-9]{1}."] + }, + { + group-ids: ["another-global-acl-group"], + fqdn-regex-list: [".*ok[0-9]{1}."] + } + ] +} + +akka { + loglevel = "INFO" + loggers = ["akka.event.slf4j.Slf4jLogger"] + logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" + logger-startup-timeout = 30s + + actor { + provider = "akka.actor.LocalActorRefProvider" + } +} + +akka.http { + server { + # The time period within which the TCP binding process must be completed. + # Set to `infinite` to disable. + bind-timeout = 5s + + # Show verbose error messages back to the client + verbose-error-messages = on + } + + parsing { + # Spray doesn't like the AWS4 headers + illegal-header-warnings = on + } +} + diff --git a/quickstart/api/logback.xml b/quickstart/api/logback.xml new file mode 100644 index 000000000..a7f4b877d --- /dev/null +++ b/quickstart/api/logback.xml @@ -0,0 +1,32 @@ + + + + + %d [test] %-5p | \(%logger{4}:%line\) | %msg %n + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/quickstart/bind9/etc/_template/named.partition.conf b/quickstart/bind9/etc/_template/named.partition.conf index 2743a5b4f..ee6ecdfe4 100644 --- a/quickstart/bind9/etc/_template/named.partition.conf +++ b/quickstart/bind9/etc/_template/named.partition.conf @@ -1,186 +1,186 @@ -zone "vinyldns{partition}" { +zone "vinyldns" { type master; - file "/var/bind/partition{partition}/vinyldns.hosts"; + file "/var/bind/partition/vinyldns.hosts"; allow-update { key "vinyldns."; }; }; zone "old-vinyldns2{partition}" { type master; - file "/var/bind/partition{partition}/old-vinyldns2.hosts"; + file "/var/bind/partition/old-vinyldns2.hosts"; allow-update { key "vinyldns."; }; }; zone "old-vinyldns3{partition}" { type master; - file "/var/bind/partition{partition}/old-vinyldns3.hosts"; + file "/var/bind/partition/old-vinyldns3.hosts"; allow-update { key "vinyldns."; }; }; -zone "dummy{partition}" { +zone "dummy" { type master; - file "/var/bind/partition{partition}/dummy.hosts"; + file "/var/bind/partition/dummy.hosts"; allow-update { key "vinyldns."; }; }; -zone "ok{partition}" { +zone "ok" { type master; - file "/var/bind/partition{partition}/ok.hosts"; + file "/var/bind/partition/ok.hosts"; allow-update { key "vinyldns."; }; }; -zone "shared{partition}" { +zone "shared" { type master; - file "/var/bind/partition{partition}/shared.hosts"; + file "/var/bind/partition/shared.hosts"; allow-update { key "vinyldns."; }; }; -zone "non.test.shared{partition}" { +zone "non.test.shared" { type master; - file "/var/bind/partition{partition}/non.test.shared.hosts"; + file "/var/bind/partition/non.test.shared.hosts"; allow-update { key "vinyldns."; }; }; -zone "system-test{partition}" { +zone "system-test" { type master; - file "/var/bind/partition{partition}/system-test.hosts"; + file "/var/bind/partition/system-test.hosts"; allow-update { key "vinyldns."; }; }; -zone "system-test-history{partition}" { +zone "system-test-history" { type master; - file "/var/bind/partition{partition}/system-test-history.hosts"; + file "/var/bind/partition/system-test-history.hosts"; allow-update { key "vinyldns."; }; }; zone "{partition}.10.in-addr.arpa" { type master; - file "/var/bind/partition{partition}/10.10.in-addr.arpa"; + file "/var/bind/partition/10.10.in-addr.arpa"; allow-update { key "vinyldns."; }; }; zone "{partition}.0.192.in-addr.arpa" { type master; - file "/var/bind/partition{partition}/2.0.192.in-addr.arpa"; + file "/var/bind/partition/2.0.192.in-addr.arpa"; allow-update { key "vinyldns."; }; }; zone "192/30.{partition}.0.192.in-addr.arpa" { type master; - file "/var/bind/partition{partition}/192^30.2.0.192.in-addr.arpa"; + file "/var/bind/partition/192^30.2.0.192.in-addr.arpa"; allow-update { key "vinyldns."; }; }; zone "{partition}.9.e.f.c.c.7.2.9.6.d.f.ip6.arpa" { type master; - file "/var/bind/partition{partition}/1.9.e.f.c.c.7.2.9.6.d.f.ip6.arpa"; + file "/var/bind/partition/1.9.e.f.c.c.7.2.9.6.d.f.ip6.arpa"; allow-update { key "vinyldns."; }; }; zone "0.0.0.1.{partition}.9.e.f.c.c.7.2.9.6.d.f.ip6.arpa" { type master; - file "/var/bind/partition{partition}/0.0.0.1.1.9.e.f.c.c.7.2.9.6.d.f.ip6.arpa"; + file "/var/bind/partition/0.0.0.1.1.9.e.f.c.c.7.2.9.6.d.f.ip6.arpa"; allow-update { key "vinyldns."; }; }; -zone "one-time{partition}" { +zone "one-time" { type master; - file "/var/bind/partition{partition}/one-time.hosts"; + file "/var/bind/partition/one-time.hosts"; allow-update { key "vinyldns."; key "vinyldns-sha1."; key "vinyldns-sha224."; key "vinyldns-sha256."; key "vinyldns-sha384."; key "vinyldns-sha512."; }; }; -zone "sync-test{partition}" { +zone "sync-test" { type master; - file "/var/bind/partition{partition}/sync-test.hosts"; + file "/var/bind/partition/sync-test.hosts"; allow-update { key "vinyldns."; }; }; -zone "invalid-zone{partition}" { +zone "invalid-zone" { type master; - file "/var/bind/partition{partition}/invalid-zone.hosts"; + file "/var/bind/partition/invalid-zone.hosts"; allow-update { key "vinyldns."; }; }; zone "list-zones-test-searched-1{partition}" { type master; - file "/var/bind/partition{partition}/list-zones-test-searched-1.hosts"; + file "/var/bind/partition/list-zones-test-searched-1.hosts"; allow-update { key "vinyldns."; }; }; zone "list-zones-test-searched-2{partition}" { type master; - file "/var/bind/partition{partition}/list-zones-test-searched-2.hosts"; + file "/var/bind/partition/list-zones-test-searched-2.hosts"; allow-update { key "vinyldns."; }; }; zone "list-zones-test-searched-3{partition}" { type master; - file "/var/bind/partition{partition}/list-zones-test-searched-3.hosts"; + file "/var/bind/partition/list-zones-test-searched-3.hosts"; allow-update { key "vinyldns."; }; }; zone "list-zones-test-unfiltered-1{partition}" { type master; - file "/var/bind/partition{partition}/list-zones-test-unfiltered-1.hosts"; + file "/var/bind/partition/list-zones-test-unfiltered-1.hosts"; allow-update { key "vinyldns."; }; }; zone "list-zones-test-unfiltered-2{partition}" { type master; - file "/var/bind/partition{partition}/list-zones-test-unfiltered-2.hosts"; + file "/var/bind/partition/list-zones-test-unfiltered-2.hosts"; allow-update { key "vinyldns."; }; }; -zone "one-time-shared{partition}" { +zone "one-time-shared" { type master; - file "/var/bind/partition{partition}/one-time-shared.hosts"; + file "/var/bind/partition/one-time-shared.hosts"; allow-update { key "vinyldns."; }; }; -zone "parent.com{partition}" { +zone "parent.com" { type master; - file "/var/bind/partition{partition}/parent.com.hosts"; + file "/var/bind/partition/parent.com.hosts"; allow-update { key "vinyldns."; }; }; -zone "child.parent.com{partition}" { +zone "child.parent.com" { type master; - file "/var/bind/partition{partition}/child.parent.com.hosts"; + file "/var/bind/partition/child.parent.com.hosts"; allow-update { key "vinyldns."; }; }; -zone "example.com{partition}" { +zone "example.com" { type master; - file "/var/bind/partition{partition}/example.com.hosts"; + file "/var/bind/partition/example.com.hosts"; allow-update { key "vinyldns."; }; }; -zone "dskey.example.com{partition}" { +zone "dskey.example.com" { type master; - file "/var/bind/partition{partition}/dskey.example.com.hosts"; + file "/var/bind/partition/dskey.example.com.hosts"; allow-update { key "vinyldns."; }; }; -zone "not.loaded{partition}" { +zone "not.loaded" { type master; - file "/var/bind/partition{partition}/not.loaded.hosts"; + file "/var/bind/partition/not.loaded.hosts"; allow-update { key "vinyldns."; }; }; -zone "zone.requires.review{partition}" { +zone "zone.requires.review" { type master; - file "/var/bind/partition{partition}/zone.requires.review.hosts"; + file "/var/bind/partition/zone.requires.review.hosts"; allow-update { key "vinyldns."; }; }; -zone "list-records{partition}" { +zone "list-records" { type master; - file "/var/bind/partition{partition}/list-records.hosts"; + file "/var/bind/partition/list-records.hosts"; allow-update { key "vinyldns."; }; }; -zone "open{partition}" { +zone "open" { type master; - file "/var/bind/partition{partition}/open.hosts"; + file "/var/bind/partition/open.hosts"; allow-update { any; }; allow-transfer { any; }; }; diff --git a/quickstart/bind9/etc/named.conf.default b/quickstart/bind9/etc/named.conf.default new file mode 100644 index 000000000..762f51787 --- /dev/null +++ b/quickstart/bind9/etc/named.conf.default @@ -0,0 +1,162 @@ +zone "vinyldns" { + type master; + file "/var/bind/default/vinyldns.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "old-vinyldns2" { + type master; + file "/var/bind/default/old-vinyldns2.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "old-vinyldns3" { + type master; + file "/var/bind/default/old-vinyldns3.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "dummy" { + type master; + file "/var/bind/default/dummy.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "ok" { + type master; + file "/var/bind/default/ok.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "shared" { + type master; + file "/var/bind/default/shared.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "non.test.shared" { + type master; + file "/var/bind/default/non.test.shared.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "system-test" { + type master; + file "/var/bind/default/system-test.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "system-test-history" { + type master; + file "/var/bind/default/system-test-history.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "10.10.in-addr.arpa" { + type master; + file "/var/bind/default/10.10.in-addr.arpa"; + allow-update { key "vinyldns."; }; + }; + +zone "one-time" { + type master; + file "/var/bind/default/one-time.hosts"; + allow-update { key "vinyldns."; key "vinyldns-sha1."; key "vinyldns-sha224."; key "vinyldns-sha256."; key "vinyldns-sha384."; key "vinyldns-sha512."; }; + }; + +zone "sync-test" { + type master; + file "/var/bind/default/sync-test.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "invalid-zone" { + type master; + file "/var/bind/default/invalid-zone.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "list-zones-test-searched-1{partition}" { + type master; + file "/var/bind/default/list-zones-test-searched-1.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "list-zones-test-searched-2{partition}" { + type master; + file "/var/bind/default/list-zones-test-searched-2.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "list-zones-test-searched-3{partition}" { + type master; + file "/var/bind/default/list-zones-test-searched-3.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "list-zones-test-unfiltered-1{partition}" { + type master; + file "/var/bind/default/list-zones-test-unfiltered-1.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "list-zones-test-unfiltered-2{partition}" { + type master; + file "/var/bind/default/list-zones-test-unfiltered-2.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "one-time-shared" { + type master; + file "/var/bind/default/one-time-shared.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "parent.com" { + type master; + file "/var/bind/default/parent.com.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "child.parent.com" { + type master; + file "/var/bind/default/child.parent.com.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "example.com" { + type master; + file "/var/bind/default/example.com.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "dskey.example.com" { + type master; + file "/var/bind/default/dskey.example.com.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "not.loaded" { + type master; + file "/var/bind/default/not.loaded.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "zone.requires.review" { + type master; + file "/var/bind/default/zone.requires.review.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "list-records" { + type master; + file "/var/bind/default/list-records.hosts"; + allow-update { key "vinyldns."; }; + }; + +zone "open" { + type master; + file "/var/bind/default/open.hosts"; + allow-update { any; }; + allow-transfer { any; }; + }; diff --git a/quickstart/bind9/etc/named.conf.local b/quickstart/bind9/etc/named.conf.local index 371b8e857..2a0fb56e4 100644 --- a/quickstart/bind9/etc/named.conf.local +++ b/quickstart/bind9/etc/named.conf.local @@ -29,6 +29,7 @@ key "vinyldns-sha512." { secret "xfKA0DYb88tiUGND+cWddwUg3/SugYSsdvCfBOJ1jr8MEdgbVRyrlVDEXLsfTUGorQ3ShENdymw2yw+rTr+lwA=="; }; +include "/etc/bind/named.conf.default"; include "/etc/bind/named.conf.partition1"; include "/etc/bind/named.conf.partition2"; include "/etc/bind/named.conf.partition3"; diff --git a/quickstart/bind9/zones/default/10.10.in-addr.arpa b/quickstart/bind9/zones/default/10.10.in-addr.arpa new file mode 100644 index 000000000..07c5b3f05 --- /dev/null +++ b/quickstart/bind9/zones/default/10.10.in-addr.arpa @@ -0,0 +1,10 @@ +$ttl 38400 +10.10.in-addr.arpa. IN SOA 172.17.42.1. admin.vinyldns.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +10.10.in-addr.arpa. IN NS 172.17.42.1. +24.0 IN PTR www.vinyl. +25.0 IN PTR mail.vinyl. diff --git a/quickstart/bind9/zones/default/child.parent.com.hosts b/quickstart/bind9/zones/default/child.parent.com.hosts new file mode 100644 index 000000000..a74630542 --- /dev/null +++ b/quickstart/bind9/zones/default/child.parent.com.hosts @@ -0,0 +1,9 @@ +$ttl 38400 +$ORIGIN child.parent.com. +@ IN SOA ns1.parent.com. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +@ IN NS ns1.parent.com. diff --git a/quickstart/bind9/zones/default/dskey.example.com.hosts b/quickstart/bind9/zones/default/dskey.example.com.hosts new file mode 100644 index 000000000..a730ac305 --- /dev/null +++ b/quickstart/bind9/zones/default/dskey.example.com.hosts @@ -0,0 +1,9 @@ +$TTL 1h +$ORIGIN dskey.example.com. +@ IN SOA ns1.parent.com. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +dskey.example.com. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/dummy.hosts b/quickstart/bind9/zones/default/dummy.hosts new file mode 100644 index 000000000..e6a53c3f1 --- /dev/null +++ b/quickstart/bind9/zones/default/dummy.hosts @@ -0,0 +1,15 @@ +$ttl 38400 +dummy. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +dummy. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 +non-approved-delegation IN NS 7.7.7.7 diff --git a/quickstart/bind9/zones/default/example.com.hosts b/quickstart/bind9/zones/default/example.com.hosts new file mode 100644 index 000000000..7e8175fd8 --- /dev/null +++ b/quickstart/bind9/zones/default/example.com.hosts @@ -0,0 +1,10 @@ +$TTL 1h +$ORIGIN example.com. +@ IN SOA ns1.parent.com. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +example.com. IN NS 172.17.42.1. +dskey IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/invalid-zone.hosts b/quickstart/bind9/zones/default/invalid-zone.hosts new file mode 100644 index 000000000..47eae6943 --- /dev/null +++ b/quickstart/bind9/zones/default/invalid-zone.hosts @@ -0,0 +1,17 @@ +$ttl 38400 +invalid-zone. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +invalid-zone. IN NS 172.17.42.1. +invalid-zone. IN NS not-approved.thing.com. +invalid.child.invalid-zone. IN NS 172.17.42.1. +dotted.host.invalid-zone. IN A 1.2.3.4 +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 diff --git a/quickstart/bind9/zones/default/list-records.hosts b/quickstart/bind9/zones/default/list-records.hosts new file mode 100644 index 000000000..f50a10fea --- /dev/null +++ b/quickstart/bind9/zones/default/list-records.hosts @@ -0,0 +1,38 @@ +$ttl 38400 +list-records. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +list-records. IN NS 172.17.42.1. +00-test-list-recordsets-0-A IN A 10.1.1.1 +00-test-list-recordsets-0-A IN A 10.2.2.2 +00-test-list-recordsets-0-CNAME IN CNAME cname1. +00-test-list-recordsets-1-A IN A 10.1.1.1 +00-test-list-recordsets-1-A IN A 10.2.2.2 +00-test-list-recordsets-1-CNAME IN CNAME cname1. +00-test-list-recordsets-2-A IN A 10.1.1.1 +00-test-list-recordsets-2-A IN A 10.2.2.2 +00-test-list-recordsets-2-CNAME IN CNAME cname1. +00-test-list-recordsets-3-A IN A 10.1.1.1 +00-test-list-recordsets-3-A IN A 10.2.2.2 +00-test-list-recordsets-3-CNAME IN CNAME cname1. +00-test-list-recordsets-4-A IN A 10.1.1.1 +00-test-list-recordsets-4-A IN A 10.2.2.2 +00-test-list-recordsets-4-CNAME IN CNAME cname1. +00-test-list-recordsets-5-A IN A 10.1.1.1 +00-test-list-recordsets-5-A IN A 10.2.2.2 +00-test-list-recordsets-5-CNAME IN CNAME cname1. +00-test-list-recordsets-6-A IN A 10.1.1.1 +00-test-list-recordsets-6-A IN A 10.2.2.2 +00-test-list-recordsets-6-CNAME IN CNAME cname1. +00-test-list-recordsets-7-A IN A 10.1.1.1 +00-test-list-recordsets-7-A IN A 10.2.2.2 +00-test-list-recordsets-7-CNAME IN CNAME cname1. +00-test-list-recordsets-8-A IN A 10.1.1.1 +00-test-list-recordsets-8-A IN A 10.2.2.2 +00-test-list-recordsets-8-CNAME IN CNAME cname1. +00-test-list-recordsets-9-A IN A 10.1.1.1 +00-test-list-recordsets-9-A IN A 10.2.2.2 +00-test-list-recordsets-9-CNAME IN CNAME cname1. diff --git a/quickstart/bind9/zones/default/list-zones-test-searched-1.hosts b/quickstart/bind9/zones/default/list-zones-test-searched-1.hosts new file mode 100644 index 000000000..c2cf966f7 --- /dev/null +++ b/quickstart/bind9/zones/default/list-zones-test-searched-1.hosts @@ -0,0 +1,8 @@ +$ttl 38400 +list-zones-test-searched-1. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +list-zones-test-searched-1. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/list-zones-test-searched-2.hosts b/quickstart/bind9/zones/default/list-zones-test-searched-2.hosts new file mode 100644 index 000000000..b531d2a19 --- /dev/null +++ b/quickstart/bind9/zones/default/list-zones-test-searched-2.hosts @@ -0,0 +1,8 @@ +$ttl 38400 +list-zones-test-searched-2. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +list-zones-test-searched-2. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/list-zones-test-searched-3.hosts b/quickstart/bind9/zones/default/list-zones-test-searched-3.hosts new file mode 100644 index 000000000..33e76e90f --- /dev/null +++ b/quickstart/bind9/zones/default/list-zones-test-searched-3.hosts @@ -0,0 +1,8 @@ +$ttl 38400 +list-zones-test-searched-3. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +list-zones-test-searched-3. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/list-zones-test-unfiltered-1.hosts b/quickstart/bind9/zones/default/list-zones-test-unfiltered-1.hosts new file mode 100644 index 000000000..9205eec0d --- /dev/null +++ b/quickstart/bind9/zones/default/list-zones-test-unfiltered-1.hosts @@ -0,0 +1,8 @@ +$ttl 38400 +list-zones-test-unfiltered-1. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +list-zones-test-unfiltered-1. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/list-zones-test-unfiltered-2.hosts b/quickstart/bind9/zones/default/list-zones-test-unfiltered-2.hosts new file mode 100644 index 000000000..dfdb66493 --- /dev/null +++ b/quickstart/bind9/zones/default/list-zones-test-unfiltered-2.hosts @@ -0,0 +1,8 @@ +$ttl 38400 +list-zones-test-unfiltered-2. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +list-zones-test-unfiltered-2. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/non.test.shared.hosts b/quickstart/bind9/zones/default/non.test.shared.hosts new file mode 100644 index 000000000..098846079 --- /dev/null +++ b/quickstart/bind9/zones/default/non.test.shared.hosts @@ -0,0 +1,13 @@ +$ttl 38400 +non.test.shared. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +non.test.shared. IN NS 172.17.42.1. +@ IN A 1.1.1.1 +delete-test-batch IN A 2.2.2.2 +update-test-batch IN A 3.3.3.3 +delete-test IN A 4.4.4.4 +update-test IN A 5.5.5.5 diff --git a/quickstart/bind9/zones/default/not.loaded.hosts b/quickstart/bind9/zones/default/not.loaded.hosts new file mode 100644 index 000000000..4f0a93779 --- /dev/null +++ b/quickstart/bind9/zones/default/not.loaded.hosts @@ -0,0 +1,9 @@ +$ttl 38400 +not.loaded. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +not.loaded. IN NS 172.17.42.1. +foo IN A 1.1.1.1 diff --git a/quickstart/bind9/zones/default/ok.hosts b/quickstart/bind9/zones/default/ok.hosts new file mode 100644 index 000000000..aaa985c5e --- /dev/null +++ b/quickstart/bind9/zones/default/ok.hosts @@ -0,0 +1,16 @@ +$ttl 38400 +ok. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +ok. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 +dotted.a IN A 7.7.7.7 +dottedc.name IN CNAME test.example.com diff --git a/quickstart/bind9/zones/default/old-shared.hosts b/quickstart/bind9/zones/default/old-shared.hosts new file mode 100644 index 000000000..a7c06b6d1 --- /dev/null +++ b/quickstart/bind9/zones/default/old-shared.hosts @@ -0,0 +1,14 @@ +$ttl 38400 +old-shared. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +old-shared. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 diff --git a/quickstart/bind9/zones/default/old-vinyldns2.hosts b/quickstart/bind9/zones/default/old-vinyldns2.hosts new file mode 100644 index 000000000..5fdc55ce9 --- /dev/null +++ b/quickstart/bind9/zones/default/old-vinyldns2.hosts @@ -0,0 +1,14 @@ +$ttl 38400 +old-vinyldns2. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +old-vinyldns2. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 diff --git a/quickstart/bind9/zones/default/old-vinyldns3.hosts b/quickstart/bind9/zones/default/old-vinyldns3.hosts new file mode 100644 index 000000000..5d514886a --- /dev/null +++ b/quickstart/bind9/zones/default/old-vinyldns3.hosts @@ -0,0 +1,14 @@ +$ttl 38400 +old-vinyldns3. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +old-vinyldns3. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 diff --git a/quickstart/bind9/zones/default/one-time-shared.hosts b/quickstart/bind9/zones/default/one-time-shared.hosts new file mode 100644 index 000000000..654f01557 --- /dev/null +++ b/quickstart/bind9/zones/default/one-time-shared.hosts @@ -0,0 +1,8 @@ +$ttl 38400 +one-time-shared. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +one-time-shared. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/one-time.hosts b/quickstart/bind9/zones/default/one-time.hosts new file mode 100644 index 000000000..df072413e --- /dev/null +++ b/quickstart/bind9/zones/default/one-time.hosts @@ -0,0 +1,14 @@ +$ttl 38400 +one-time. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +one-time. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 diff --git a/quickstart/bind9/zones/default/open.hosts b/quickstart/bind9/zones/default/open.hosts new file mode 100644 index 000000000..48f994103 --- /dev/null +++ b/quickstart/bind9/zones/default/open.hosts @@ -0,0 +1,8 @@ +$ttl 38400 +open. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +open. IN NS 172.17.42.1. diff --git a/quickstart/bind9/zones/default/parent.com.hosts b/quickstart/bind9/zones/default/parent.com.hosts new file mode 100644 index 000000000..c3dc749f6 --- /dev/null +++ b/quickstart/bind9/zones/default/parent.com.hosts @@ -0,0 +1,15 @@ +$ttl 38400 +$ORIGIN parent.com. +@ IN SOA ns1.parent.com. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +parent.com. IN NS ns1.parent.com. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +already-exists IN A 6.6.6.6 +ns1 IN A 172.17.42.1 diff --git a/quickstart/bind9/zones/default/shared.hosts b/quickstart/bind9/zones/default/shared.hosts new file mode 100644 index 000000000..d9115a129 --- /dev/null +++ b/quickstart/bind9/zones/default/shared.hosts @@ -0,0 +1,16 @@ +$ttl 38400 +shared. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +shared. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 +delete-test-batch IN A 2.2.2.2 +update-test-batch IN A 3.3.3.3 diff --git a/quickstart/bind9/zones/default/sync-test.hosts b/quickstart/bind9/zones/default/sync-test.hosts new file mode 100644 index 000000000..a369f01dd --- /dev/null +++ b/quickstart/bind9/zones/default/sync-test.hosts @@ -0,0 +1,17 @@ +$ttl 38400 +sync-test. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +sync-test. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 +fqdn.sync-test. IN A 7.7.7.7 +_sip._tcp IN SRV 10 60 5060 foo.sync-test. +existing.dotted IN A 9.9.9.9 diff --git a/quickstart/bind9/zones/default/system-test-history.hosts b/quickstart/bind9/zones/default/system-test-history.hosts new file mode 100644 index 000000000..1408efda6 --- /dev/null +++ b/quickstart/bind9/zones/default/system-test-history.hosts @@ -0,0 +1,14 @@ +$ttl 38400 +system-test-history. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +system-test-history. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 diff --git a/quickstart/bind9/zones/default/system-test.hosts b/quickstart/bind9/zones/default/system-test.hosts new file mode 100644 index 000000000..75a819a33 --- /dev/null +++ b/quickstart/bind9/zones/default/system-test.hosts @@ -0,0 +1,16 @@ +$ttl 38400 +system-test. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +system-test. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 +high-value-domain IN A 1.1.1.1 +high-VALUE-domain-UPPER-CASE IN A 1.1.1.1 diff --git a/quickstart/bind9/zones/default/vinyldns.hosts b/quickstart/bind9/zones/default/vinyldns.hosts new file mode 100644 index 000000000..905211823 --- /dev/null +++ b/quickstart/bind9/zones/default/vinyldns.hosts @@ -0,0 +1,14 @@ +$ttl 38400 +vinyldns. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +vinyldns. IN NS 172.17.42.1. +jenkins IN A 10.1.1.1 +foo IN A 2.2.2.2 +test IN A 3.3.3.3 +test IN A 4.4.4.4 +@ IN A 5.5.5.5 +already-exists IN A 6.6.6.6 diff --git a/quickstart/bind9/zones/default/zone.requires.review.hosts b/quickstart/bind9/zones/default/zone.requires.review.hosts new file mode 100644 index 000000000..b1deedda4 --- /dev/null +++ b/quickstart/bind9/zones/default/zone.requires.review.hosts @@ -0,0 +1,11 @@ +$ttl 38400 +zone.requires.review. IN SOA 172.17.42.1. admin.test.com. ( + 1439234395 + 10800 + 3600 + 604800 + 38400 ) +zone.requires.review. IN NS 172.17.42.1. +@ IN A 1.1.1.1 +delete-test-batch IN A 2.2.2.2 +update-test-batch IN A 3.3.3.3 diff --git a/quickstart/docker-compose.yml b/quickstart/docker-compose.yml index 2422a916f..f0bdf785d 100644 --- a/quickstart/docker-compose.yml +++ b/quickstart/docker-compose.yml @@ -3,41 +3,61 @@ version: "3.5" services: ldap: container_name: "vinyldns-ldap" - image: rroemhild/test-openldap + image: vinyldns/build:openldap ports: - - "19004:389" + - "19004:19004" integration: container_name: "vinyldns-api-integration" - hostname: "vinyldns-integration" - image: "vinyldns-api-integration" + hostname: &integration_hostname "vinyldns-integration" + image: "vinyldns/build:base-test-integration-${VINYLDNS_IMAGE_VERSION}" build: context: ../ dockerfile: test/api/integration/Dockerfile + args: + VINYLDNS_VERSION: "${VINYLDNS_VERSION}" environment: - RUN_SERVICES: "all tail-logs" + RUN_SERVICES: "deps-only tail-logs" + LOCALSTACK_EXT_HOSTNAME: *integration_hostname + env_file: + .env + ports: + - "19001-19003:19001-19003/tcp" + - "19001:19001/udp" + + api: + container_name: "vinyldns-api" + image: "vinyldns/api:${VINYLDNS_IMAGE_VERSION}" + build: + context: ../ + dockerfile: quickstart/api/Dockerfile + args: + VINYLDNS_VERSION: "${VINYLDNS_VERSION}" + DOCKER_FILE_PATH: "quickstart/api" + volumes: + - ./api/application.conf:/opt/vinyldns/conf/vinyldns.conf env_file: .env ports: - "9000:9000" - - "19001-19003:19001-19003/tcp" - - "19001:19001/udp" + depends_on: + - integration portal: container_name: "vinyldns-portal" - image: "vinyldns/portal:${VINYLDNS_VERSION}" + image: "vinyldns/portal:${VINYLDNS_IMAGE_VERSION}" build: - context: .. - dockerfile: "" + context: ../ + dockerfile: quickstart/portal/Dockerfile env_file: .env ports: - "${PORTAL_PORT}:${PORTAL_PORT}" volumes: - - ./portal/application.ini:/opt/docker/conf/application.ini - - ./portal/application.conf:/opt/docker/conf/application.conf + - ./portal/application.ini:/opt/vinyldns/conf/application.ini + - ./portal/application.conf:/opt/vinyldns/conf/application.conf depends_on: - - integration + - api - ldap networks: diff --git a/quickstart/portal/Dockerfile b/quickstart/portal/Dockerfile index a2b2302fc..03f701525 100644 --- a/quickstart/portal/Dockerfile +++ b/quickstart/portal/Dockerfile @@ -13,22 +13,22 @@ RUN sbt "set version in ThisBuild := \"${VINYLDNS_VERSION}\"; project portal; un FROM adoptopenjdk/openjdk11:jdk-11.0.8_10-alpine -RUN apk add --update --no-cache netcat-openbsd bash +RUN apk add --update --no-cache bash COPY --from=builder /vinyldns/modules/portal/target/universal/portal.tgz / RUN mkdir -p /opt && \ tar -xzvf /portal.tgz && \ - mv /portal /opt/docker && \ - mkdir -p /opt/docker/lib_extra + mv /portal /opt/vinyldns && \ + mkdir -p /opt/vinyldns/lib_extra # This will set the vinyldns version, make sure to have this in config... version = ${?VINYLDNS_VERSION} ENV VINYLDNS_VERSION=$VINYLDNS_VERSION # Mount the volume for config file and lib extras # Note: These volume names are used in the build.sbt -VOLUME ["/opt/docker/lib_extra/", "/opt/docker/conf"] +VOLUME ["/opt/vinyldns/lib_extra/", "/opt/vinyldns/conf"] -EXPOSE 9000 +EXPOSE 9001 -ENTRYPOINT ["/opt/docker/bin/portal"] +ENTRYPOINT ["/opt/vinyldns/bin/portal"] diff --git a/quickstart/portal/Makefile b/quickstart/portal/Makefile index 8f0197073..0a1867da3 100644 --- a/quickstart/portal/Makefile +++ b/quickstart/portal/Makefile @@ -1,5 +1,5 @@ SHELL=bash -IMAGE_NAME=vinyldns/portal +IMAGE_NAME=vinyldns/portal:local-dev ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) # Check that the required version of make is being used @@ -17,6 +17,9 @@ ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif +ifdef $(WITH_ARGS) + ARG_SEPARATOR=-- +endif %: @: @@ -30,11 +33,11 @@ all: build run build: @set -euo pipefail cd ../.. - docker build -t $(IMAGE_NAME) --build-arg DOCKERFILE_PATH="$(RELATIVE_ROOT_DIR)" -f "$(ROOT_DIR)/Dockerfile" . + docker build -t $(IMAGE_NAME) -f "$(ROOT_DIR)/Dockerfile" . run: @set -euo pipefail - docker run -it --rm $(DOCKER_PARAMS) -p 9001:9001 $(IMAGE_NAME) -- $(WITH_ARGS) + docker run -it --rm $(DOCKER_PARAMS) -p 9001:9001 $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) run-bg: @set -euo pipefail diff --git a/quickstart/quickstart-vinyldns.sh b/quickstart/quickstart-vinyldns.sh new file mode 100644 index 000000000..18ff241af --- /dev/null +++ b/quickstart/quickstart-vinyldns.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +##################################################################################################### +# Starts up the api, portal, and dependent services via +# docker-compose. The api will be available on localhost:9000 and the +# portal will be on localhost:9001 +# +# Relevant overrides can be found in .env +##################################################################################################### +set -eo pipefail + +DIR=$( cd "$(dirname "$0")" || exit ; pwd -P ) +source "${DIR}/../utils/includes/terminal_colors.sh" + +function usage() { + echo -e "usage: quickstart-vinyldns.sh [OPTIONS]" + echo -e "Starts up a local VinylDNS installation using Docker Compose" + echo + echo -e "options:" + echo -e "\t-a, --api-only do not start up the VinylDNS Portal" + echo -e "\t-b, --build force a rebuild of the Docker images with the local code" + echo -e "\t-r, --reset reset any the running containers" + echo -e "\t-s, --service specify the service to run" + echo -e "\t-t, --timeout the time to wait (in seconds) for the portal and API to start (default: 60)" + echo -e "\t-u, --update remove the local quickstart images to force a re-pull from docker hub" + echo -e "\t-v, --version-tag specify Docker image tag version (default: latest)" + echo + echo -e "\t-h, --help show this help" +} + +function wait_for_url() { + echo -n "Waiting for ${F_BLUE}$1${F_RESET} at ${URL}.." + RETRY="$TIMEOUT" + while [ "$RETRY" -ge 0 ]; do + echo -n "." + if curl -I -s "${URL}" -o /dev/null -w "%{http_code}" &>/dev/null || false; then + echo "${F_GREEN}OK${F_RESET}" + break + else + ((RETRY -= 1)) + sleep 1 + if [[ $RETRY -eq 1 ]]; then + echo "${F_RED}FAILED${F_RESET}" + echo "${F_RED}Timeout waiting for ${F_BLUE}$1${F_RED} to be ready${F_RESET}" + exit 1 + fi + fi + done +} + +function wait_for_api() { + URL="$VINYLDNS_API_URL" + wait_for_url "VinylDNS API" +} + +function wait_for_portal() { + # check if portal was skipped + if [ "$SERVICE" != "api" ]; then + URL="$VINYLDNS_PORTAL_URL" + wait_for_url "VinylDNS Portal" + fi +} + +# Source customizable env files ('-a' causes all variables to be exported) +set -a; source "${DIR}/.env"; set +a + +# Set defaults and parse args +export VINYLDNS_VERSION=latest +TIMEOUT=60 +DOCKER_COMPOSE_CONFIG="${DIR}/docker-compose.yml" +SERVICE="" +BUILD="" +RESET_DOCKER=0 +UPDATE=0 +while [[ $# -gt 0 ]]; do + case "$1" in + -t | --timeout) + TIMEOUT="$2" + shift + shift + ;; + -a | --api-only) + SERVICE="api" + shift + ;; + -s | --service) + SERVICE="$2" + shift + shift + ;; + -u | --update) + UPDATE=1 + shift + ;; + -b | --build) + BUILD="--build" + shift + ;; + -r | --reset) + RESET_DOCKER=1 + shift + ;; + -v | --version-tag) + export VINYLDNS_VERSION=$2 + shift + shift + ;; + *) + usage + exit + ;; + esac +done + +if [[ $RESET_DOCKER -eq 1 ]]; then + "${DIR}/../utils/clean-vinyldns-containers.sh" +fi + +export VINYLDNS_IMAGE_VERSION=${VINYLDNS_VERSION} +if [ -n "${BUILD}" ] || [ -n "$(docker images vinyldns/portal:local-dev --format '{{.Repository}}:{{.Tag}}')" ]; then + VINYLDNS_IMAGE_VERSION="local-dev" +fi + +# Update images if requested +if [[ $UPDATE -eq 1 ]]; then + echo "${F_YELLOW}Removing local docker images tagged ${F_RESET}'${VINYLDNS_IMAGE_VERSION}'${F_YELLOW}...${F_RESET}" + "${DIR}/../utils/clean-vinyldns-containers.sh" + docker rmi "vinyldns/build:base-test-integration-${VINYLDNS_IMAGE_VERSION}" + docker rmi "vinyldns/portal:${VINYLDNS_IMAGE_VERSION}" + docker rmi "vinyldns/api:${VINYLDNS_IMAGE_VERSION}" +fi + +if [ -n "${BUILD}" ]; then + echo "Building containers and starting VinylDNS (${VINYLDNS_IMAGE_VERSION}) in the background..." +else + echo "Starting VinylDNS (${VINYLDNS_IMAGE_VERSION}) the background..." +fi +docker-compose -f "${DOCKER_COMPOSE_CONFIG}" up ${BUILD} -d ${SERVICE} || (echo -e "${F_RED}Sorry, there was an error starting VinylDNS :-(\nTry resetting any existing containers with:\n\t${F_RESET}'$0 --reset'"; exit 1) + +echo +wait_for_api +wait_for_portal +echo + +if [ "${SERVICE}" != "api" ]; then + echo "${F_GREEN}VinylDNS started. You can connect to the portal via ${F_RESET}${VINYLDNS_PORTAL_URL}" +else + echo "${F_GREEN}VinylDNS API started. You can connect to the API via ${F_RESET}${VINYLDNS_API_URL}" +fi +echo "${F_GREEN}To clean up the running containers:${F_RESET}" +echo " $0 --reset" diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..922776874 --- /dev/null +++ b/test/README.md @@ -0,0 +1,20 @@ +# Test + +This folder contains test containers for running integration and functional tests. + +| Path |Description | +| --- | --- | +|`api/functional` | A Docker container for running functional tests. Use the `Makefile` to build and run.
You can use `WITH_ARGS` to pass [arguments to `Pytest`](https://docs.pytest.org/en/6.2.x/usage.html#specifying-tests-selecting-tests).
Ex: `make run WITH_ARGS="-k test_verify_production"`

The tests are located in `modules/api/test/functional`| +|`api/integration` | A Docker container for running integration tests. Use the `Makefile` to build and run.
You can use `WITH_ARGS` to pass the command you would like to execute in the context of this container.
Ex: `make run WITH_ARGS="sbt ';validate'"`

This does not run any tests by default, but is used by other scripts to run the integration tests. (e.g., `build/verify.sh`)| +|`portal/functional` | A Docker container for running functional tests. Use the `Makefile` to build and run.

The tests are located in `modules/portal/test`| + +## Execution + +The functional tests can be run by using `make` as described in the table above. For other usages, check out +the `build/` directory. Specifically: + +| Path |Description | +| --- | --- | +| `build/func-test-api.sh` | Runs the functional tests for the API| +| `build/func-test-portal.sh` | Runs the functional tests for the Portal| +| `build/verify.sh` | Runs the unit and integration tests for everything| diff --git a/test/api/functional/Dockerfile b/test/api/functional/Dockerfile index 3ea913b81..058638172 100644 --- a/test/api/functional/Dockerfile +++ b/test/api/functional/Dockerfile @@ -20,11 +20,11 @@ ARG DOCKERFILE_PATH COPY --from=base-build /opt/vinyldns /opt/vinyldns # Local bind server files -COPY docker/bind9/etc/named.conf.* /etc/bind/ -COPY docker/bind9/zones/ /var/bind/ +COPY quickstart/bind9/etc/named.conf.* /etc/bind/ +COPY quickstart/bind9/zones/ /var/bind/ RUN named-checkconf # Copy over the functional tests -COPY ${DOCKERFILE_PATH}/test /functional_test +COPY modules/api/src/test/functional /functional_test ENTRYPOINT ["/bin/bash", "-c", "/initialize.sh bind localstack vinyldns-api && /functional_test/run.sh \"$@\""] diff --git a/test/api/functional/Makefile b/test/api/functional/Makefile index 810205023..7460bba99 100644 --- a/test/api/functional/Makefile +++ b/test/api/functional/Makefile @@ -13,7 +13,7 @@ endif # Extract arguments for `make run` EXTRACT_ARGS=true ifeq (run,$(firstword $(MAKECMDGOALS))) - EXTRACT_ARGS=true + EXTRACT_ARGS=true endif ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" @@ -21,7 +21,9 @@ ifeq ($(EXTRACT_ARGS),true) # ...and turn them into do-nothing targets $(eval $(WITH_ARGS):;@:) endif - +ifdef WITH_ARGS + ARG_SEPARATOR=-- +endif .ONESHELL: @@ -31,15 +33,13 @@ all: build run build: @set -euo pipefail - trap 'if [ -f "$(ROOT_DIR)/vinyldns.jar" ]; then rm $(ROOT_DIR)/vinyldns.jar; fi' EXIT cd ../../.. - if [ -f modules/api/target/scala-2.12/vinyldns.jar ]; then cp modules/api/target/scala-2.12/vinyldns.jar $(ROOT_DIR)/vinyldns.jar; fi docker build -t $(IMAGE_NAME) $(DOCKER_PARAMS)--build-arg DOCKERFILE_PATH="$(RELATIVE_ROOT_DIR)" -f "$(ROOT_DIR)/Dockerfile" . run: @set -euo pipefail - docker run -it --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp $(IMAGE_NAME) -- $(WITH_ARGS) + docker run -it --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) run-local: @set -euo pipefail - docker run -it --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp -v "$$(pwd)/test:/functional_test" $(IMAGE_NAME) -- $(WITH_ARGS) + docker run -it --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp -v "$(ROOT_DIR)/../../../modules/api/src/test/functional:/functional_test" $(IMAGE_NAME) -- $(WITH_ARGS) diff --git a/test/api/functional/vinyldns.conf b/test/api/functional/vinyldns.conf index a47ad700b..5b02a3601 100644 --- a/test/api/functional/vinyldns.conf +++ b/test/api/functional/vinyldns.conf @@ -183,7 +183,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.2.252", "192.0.2.253", "fd69:27cc:fe91:0:0:0:0:ffff", @@ -233,7 +232,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.1.252", "192.0.1.253", "192.0.2.252", diff --git a/test/api/integration/Dockerfile b/test/api/integration/Dockerfile index 0d07e335b..ffa44f6cc 100644 --- a/test/api/integration/Dockerfile +++ b/test/api/integration/Dockerfile @@ -1,5 +1,6 @@ # Build VinylDNS API if the JAR doesn't already exist -FROM vinyldns/build:base-build as base-build +ARG VINYLDNS_VERSION=latest +FROM vinyldns/build:base-build-${VINYLDNS_VERSION} as base-build ARG DOCKERFILE_PATH="test/api/integration" COPY "${DOCKERFILE_PATH}/vinyldns.*" /opt/vinyldns/ COPY . /build/ @@ -10,11 +11,11 @@ RUN if [ -f assembly/vinyldns.jar ]; then cp assembly/vinyldns.jar /opt/vinyldns if [ ! -f /opt/vinyldns/vinyldns.jar ]; then \ env SBT_OPTS="-XX:+UseConcMarkSweepGC -Xmx4G -Xms1G" \ sbt -Dbuild.scalafmtOnCompile=false -Dbuild.lintOnCompile=fase ";project api;coverageOff;assembly" \ - && cp modules/api/target/scala-2.12/vinyldns.jar /opt/vinyldns/; \ + && cp assembly/vinyldns.jar /opt/vinyldns/; \ fi # Build the testing image, copying data from `vinyldns-api` -FROM vinyldns/build:base-test-integration +FROM vinyldns/build:base-test-integration-${VINYLDNS_VERSION} SHELL ["/bin/bash","-c"] ARG DOCKERFILE_PATH COPY --from=base-build /opt/vinyldns /opt/vinyldns @@ -24,8 +25,8 @@ COPY . /build/ WORKDIR /build # Local bind server files -COPY docker/bind9/etc/named.conf.* /etc/bind/ -COPY docker/bind9/zones/ /var/bind/ +COPY quickstart/bind9/etc/named.conf.* /etc/bind/ +COPY quickstart/bind9/zones/ /var/bind/ RUN named-checkconf ENV RUN_SERVICES="all" diff --git a/test/api/integration/Makefile b/test/api/integration/Makefile index 6e0fd3e0d..a1c0ed918 100644 --- a/test/api/integration/Makefile +++ b/test/api/integration/Makefile @@ -18,6 +18,9 @@ ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif +ifdef WITH_ARGS + ARG_SEPARATOR=-- +endif %: @: @@ -35,7 +38,7 @@ build: run: @set -euo pipefail - docker run -it --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp $(IMAGE_NAME) -- $(WITH_ARGS) + docker run -it --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) run-bg: @set -euo pipefail diff --git a/test/api/integration/vinyldns.conf b/test/api/integration/vinyldns.conf index a47ad700b..a13a8b256 100644 --- a/test/api/integration/vinyldns.conf +++ b/test/api/integration/vinyldns.conf @@ -3,6 +3,35 @@ ################################################################################################################ vinyldns { + base-version = "0.0.0-local-dev" + version = ${vinyldns.base-version} # default to the base version if not overridden + version = ${?VINYLDNS_VERSION} # override the base version via env var + + # How often to any particular zone can be synchronized in milliseconds + sync-delay = 10000 + sync-delay = ${?SYNC_DELAY} + + # If we should start up polling for change requests, set this to false for the inactive cluster + processing-disabled = false + processing-disabled = ${?PROCESSING_DISABLED} + + # Number of records that can be in a zone + max-zone-size = 60000 + max-zone-size = ${?MAX_ZONE_SIZE} + + # Types of unowned records that users can access in shared zones + shared-approved-types = ["A", "AAAA", "CNAME", "PTR", "TXT"] + + # Batch change settings + batch-change-limit = 1000 + batch-change-limit = ${?BATCH_CHANGE_LIMIT} + manual-batch-review-enabled = true + manual-batch-review-enabled = ${?MANUAL_BATCH_REVIEW_ENABLED} + scheduled-changes-enabled = true + scheduled-changes-enabled = ${?SCHEDULED_CHANGES_ENABLED} + multi-record-batch-change-enabled = true + multi-record-batch-change-enabled = ${?MULTI_RECORD_BATCH_CHANGE_ENABLED} + # configured backend providers backend { # Use "default" when dns backend legacy = true @@ -93,9 +122,35 @@ vinyldns { } } + email { + class-name = "vinyldns.api.notifier.email.EmailNotifierProvider" + class-name = ${?EMAIL_CLASS_NAME} + settings = { + from = "VinylDNS " + } + } + + sns { + class-name = "vinyldns.api.notifier.sns.SnsNotifierProvider" + class-name = ${?SNS_CLASS_NAME} + settings { + topic-arn = "arn:aws:sns:us-east-1:000000000000:batchChanges" + topic-arn = ${?SNS_TOPIC_ARN} + access-key = "test" + access-key = ${?SNS_ACCESS_KEY} + secret-key = "test" + secret-key = ${?SNS_SECRET_KEY} + service-endpoint = "http://127.0.0.1:19003" + service-endpoint = ${?SNS_SERVICE_ENDPOINT} + signing-region = "us-east-1" + signing-region = ${?SNS_REGION} + } + } + rest { host = "0.0.0.0" port = 9000 + port=${?API_SERVICE_PORT} } sync-delay = 10000 @@ -121,6 +176,7 @@ vinyldns { # these must be overridden to use MYSQL for production use # assumes a docker or mysql instance running locally name = "vinyldns" + name = ${?DATABASE_NAME} driver = "org.h2.Driver" driver = ${?JDBC_DRIVER} migration-url = "jdbc:h2:mem:vinyldns;MODE=MYSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE;IGNORECASE=TRUE;INIT=RUNSCRIPT FROM 'classpath:test/ddl.sql'" @@ -183,7 +239,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.2.252", "192.0.2.253", "fd69:27cc:fe91:0:0:0:0:ffff", @@ -233,7 +288,6 @@ vinyldns { "high-value-domain.*" # for testing ] ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing "192.0.1.252", "192.0.1.253", "192.0.2.252", diff --git a/test/portal/functional/Makefile b/test/portal/functional/Makefile index 74ada4fd4..926d37314 100644 --- a/test/portal/functional/Makefile +++ b/test/portal/functional/Makefile @@ -20,7 +20,9 @@ ifeq ($(EXTRACT_ARGS),true) # ...and turn them into do-nothing targets $(eval $(WITH_ARGS):;@:) endif - +ifdef WITH_ARGS + ARG_SEPARATOR=-- +endif .ONESHELL: @@ -39,4 +41,4 @@ run: run-local: @set -euo pipefail - docker run -it --rm -v "$$(pwd)/../../../modules/portal:/functional_test" $(IMAGE_NAME) -- $(WITH_ARGS) + docker run -it --rm -v "$$(pwd)/../../../modules/portal:/functional_test" $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) diff --git a/utils/clean-vinyldns-containers.sh b/utils/clean-vinyldns-containers.sh index 9e2e90199..3541a8bbb 100755 --- a/utils/clean-vinyldns-containers.sh +++ b/utils/clean-vinyldns-containers.sh @@ -6,19 +6,19 @@ # Note: this will not remove the actual images from your # machine, just the running containers -ALL_IDS=$(docker ps -a | grep -e 'vinyldns' -e 'flaviovs/mock-smtp' -e 'rroemhild/test-openldap' | awk '{print $1}') +ALL_IDS=$(docker ps -a | grep -e 'vinyldns' -e 'flaviovs/mock-smtp' | awk '{print $1}') if [ "${ALL_IDS}" == "" ]; then echo "Nothing to remove" exit 0 fi -RUNNING_IDS=$(docker ps | grep -e 'vinyldns' -e 'flaviovs/mock-smtp' -e 'rroemhild/test-openldap' | awk '{print $1}') +RUNNING_IDS=$(docker ps | grep -e 'vinyldns' -e 'flaviovs/mock-smtp' | awk '{print $1}') if [ "${RUNNING_IDS}" != "" ]; then echo "Killing running containers..." echo "${RUNNING_IDS}" | xargs docker kill fi -ALL_IDS=$(docker ps -a | grep -e 'vinyldns' -e 'flaviovs/mock-smtp' -e 'rroemhild/test-openldap' | awk '{print $1}') +ALL_IDS=$(docker ps -a | grep -e 'vinyldns' -e 'flaviovs/mock-smtp' | awk '{print $1}') if [ "${ALL_IDS}" != "" ]; then echo "Removing containers..." echo "${ALL_IDS}" | xargs docker rm -v diff --git a/utils/includes/terminal_colors.sh b/utils/includes/terminal_colors.sh new file mode 100644 index 000000000..be7d81875 --- /dev/null +++ b/utils/includes/terminal_colors.sh @@ -0,0 +1,18 @@ +# Color codes prefixed with F_ (for "font") to disambiguate +F_RESET=$(echo -en '\033[0m') +F_RED=$(echo -en '\033[00;31m') +F_GREEN=$(echo -en '\033[00;32m') +F_YELLOW=$(echo -en '\033[00;33m') +F_BLUE=$(echo -en '\033[00;34m') +F_MAGENTA=$(echo -en '\033[00;35m') +F_PURPLE=$(echo -en '\033[00;35m') +F_CYAN=$(echo -en '\033[00;36m') +F_LGRAY=$(echo -en '\033[00;37m') +F_LRED=$(echo -en '\033[01;31m') +F_LGREEN=$(echo -en '\033[01;32m') +F_LYELLOW=$(echo -en '\033[01;33m') +F_LBLUE=$(echo -en '\033[01;34m') +F_LMAGENTA=$(echo -en '\033[01;35m') +F_LPURPLE=$(echo -en '\033[01;35m') +F_LCYAN=$(echo -en '\033[01;36m') +F_WHITE=$(echo -en '\033[01;37m') diff --git a/utils/quickstart-vinyldns.sh b/utils/quickstart-vinyldns.sh deleted file mode 100755 index 37dd5219a..000000000 --- a/utils/quickstart-vinyldns.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash -##################################################################################################### -# Starts up the api, portal, and dependent services via -# docker-compose. The api will be available on localhost:9000 and the -# portal will be on localhost:9001 -# -# Relevant overrides can be found in quickstart/.env -# -# Options: -# -t, --timeout seconds: overwrite ping timeout of 60 -# -a, --api-only: do not start up vinyldns-portal -# -s, --service: specify the service to run -# -c, --clean: re-pull vinyldns/api and vinyldns/portal images from docker hub -# -b, --build: rebuild images when applicable -# -v, --version tag: overwrite vinyldns/api and vinyldns/portal docker tags -##################################################################################################### -set -eo pipefail - -function wait_for_url() { - echo -n "Checking ${URL}..." - RETRY="$TIMEOUT" - while [ "$RETRY" -gt 0 ]; do - if curl -I -s "${URL}" -o /dev/null -w "%{http_code}" &>/dev/null || false; then - echo "Succeeded in connecting to ${URL}!" - break - else - echo -n "." - - ((RETRY -= 1)) - sleep 1 - - if [ "$RETRY" -eq 0 ]; then - echo "Exceeded retries waiting for ${URL} to be ready, failing" - exit 1 - fi - fi - done -} - -function usage() { - printf "usage: quickstart-vinyldns.sh [OPTIONS]\n\n" - printf "Starts up a local VinylDNS installation using docker compose\n\n" - printf "options:\n" - printf "\t-t, --timeout seconds: overwrite ping timeout of 60\n" - printf "\t-a, --api-only: do not start up vinyldns-portal\n" - printf "\t-s, --service: specify the service to run\n" - printf "\t-c, --clean: re-pull vinyldns/api and vinyldns/portal images from docker hub\n" - printf "\t-b, --build: rebuild images when applicable\n" - printf "\t-v, --version tag: overwrite vinyldns/api and vinyldns/portal docker tags\n" -} - -function clean_images() { - if [[ $CLEAN -eq 1 ]]; then - echo "cleaning docker images..." - docker rmi "vinyldns/api:${VINYLDNS_VERSION}" - docker rmi "vinyldns/portal:${VINYLDNS_VERSION}" - fi -} - -function wait_for_api() { - echo "Waiting for api..." - URL="$VINYLDNS_API_URL" - wait_for_url -} - -function wait_for_portal() { - # check if portal was skipped - if [ "$SERVICE" != "integration" ]; then - echo "Waiting for portal..." - URL="$VINYLDNS_PORTAL_URL" - wait_for_url - fi -} - -# initial var setup -DIR=$( - cd "$(dirname "$0")" - pwd -P -) -TIMEOUT=60 -DOCKER_COMPOSE_CONFIG="${DIR}/../quickstart/docker-compose.yml" -# empty service starts up all docker services in compose file -SERVICE="" -# when CLEAN is set to 1, existing docker images are deleted so they are re-pulled -CLEAN=0 -# default to latest for docker versions -export VINYLDNS_VERSION=latest - -# source env before parsing args so vars can be overwritten -set -a # Required in order to source docker/.env -# Source customizable env files -source "$DIR"/../quickstart/.env - -# parse args -BUILD="" -while [[ $# -gt 0 ]]; do - case "$1" in - -t | --timeout) - TIMEOUT="$2" - shift - shift - ;; - -a | --api-only) - SERVICE="integration" - shift - ;; - -s | --service) - SERVICE="$2" - shift - shift - ;; - -c | --clean) - CLEAN=1 - shift - ;; - -b | --build) - BUILD="--build" - shift - ;; - -v | --version) - export VINYLDNS_VERSION=$2 - shift - shift - ;; - *) - usage - exit - ;; - esac -done - -clean_images - -echo "timeout is set to ${TIMEOUT}" -echo "vinyldns version is set to '${VINYLDNS_VERSION}'" - -echo "Starting vinyldns and all dependencies in the background..." -docker-compose -f "$DOCKER_COMPOSE_CONFIG" up ${BUILD} -d "${SERVICE}" - -wait_for_api -wait_for_portal diff --git a/utils/release.sh b/utils/release.sh deleted file mode 100755 index da9ed02f2..000000000 --- a/utils/release.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -printf "\nnote: follow the guides in MAINTAINERS.md to setup notary delegation (Docker) and get sonatype key (Maven) \n" - -DIR=$( cd $(dirname $0) ; pwd -P ) - -# gpg sbt plugin fails if this is not set -export GPG_TTY=$(tty) - -## -# running tests -## -if [ "$1" != "skip-tests" ]; then - # Checking for uncommitted changes - printf "\nchecking for uncommitted changes... \n" - if ! (cd "$DIR" && git add . && git diff-index --quiet HEAD --) - then - printf "\nerror: attempting to release with uncommitted changes\n" - exit 1 - fi - # If we are not in the main repository then fail fast - REMOTE_REPO=$(git config --get remote.origin.url) - echo "REMOTE REPO IS $REMOTE_REPO" - if [[ "$REMOTE_REPO" != *-vinyldns/vinyldns.git ]]; then - printf "\nCannot run a release from this repository as it is not the main repository: $REMOTE_REPO \n" - exit 1 - fi - - # If we are not on the master branch,then fail fast - BRANCH=$(git rev-parse --abbrev-ref HEAD) - if [[ "$BRANCH" != "master" ]]; then - printf "\nCannot run a release from this branch: $BRANCH is not master \n" - exit 1; - fi - - printf "\nrunning api func tests... \n" - if ! "$DIR"/func-test-api.sh - then - printf "\nerror: utils/func-test-api.sh failed \n" - exit 1 - fi - - printf "\nrunning portal func tests... \n" - if ! "$DIR"/func-test-portal.sh - then - printf "\nerror: utils/func-test-portal.sh failed \n" - exit 1 - fi - - printf "\nrunning verify... \n" - if ! "$DIR"/verify.sh - then - printf "\nerror: utils/verify.sh failed \n" - exit 1 - fi -fi - -## -# run release -## -cd "$DIR"/../ && sbt release && cd $DIR - -printf "\nrelease finished \n" diff --git a/utils/verify.sh b/utils/verify.sh deleted file mode 100755 index f77cd4a35..000000000 --- a/utils/verify.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) -echo 'Running tests...' - -cd "$DIR/../test/api/integration" -make build && make run WITH_ARGS="sbt ';validate;verify'" -verify_result=$? - -if [ ${verify_result} -eq 0 ]; then - echo 'Verify successful!' - exit 0 -else - echo 'Verify failed!' - exit 1 -fi diff --git a/version.sbt b/version.sbt index 3da2cab98..02ae30c06 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.9.10" +version in ThisBuild := "0.10.0"