From e1743e5342e7ac146fa8a1e3f148907de759af7a Mon Sep 17 00:00:00 2001 From: "Emerle, Ryan" Date: Tue, 2 Nov 2021 17:06:24 -0400 Subject: [PATCH] Updates - Simplify build config - Add TTY check to Makefiles for running Docker containers - Update `fs2` to latest patch - Update `sbt-assembly` plugin - Update portal to remove chatty console - Update portal scripts to add license header - Update prepare-portal/Gruntfile to combine js and css where applicable - Remove unused gentelella files from final portal artifact - Add support for shared zones to quickstart/docker images - Consolidate built artifacts in `artifacts/` to make eventual release easier --- .dockerignore | 1 + .gitignore | 2 +- README.md | 58 +-- build.sbt | 115 +++--- .../{assemble_api_jar.sh => assemble_api.sh} | 9 +- build/assemble_portal.sh | 50 +++ build/docker/README.md | 8 +- build/docker/api/Dockerfile | 15 +- build/docker/api/Makefile | 8 +- build/docker/api/application.conf | 49 +-- build/docker/portal/Dockerfile | 56 +-- build/docker/portal/Makefile | 10 +- build/docker/portal/application.conf | 30 +- build/docker/portal/logback.xml | 21 ++ build/run_all_tests.sh | 4 +- modules/docs/Makefile | 2 +- .../docs/src/main/mdoc/operator/setup-api.md | 4 +- .../src/main/mdoc/operator/setup-portal.md | 33 +- modules/portal/.gitignore | 13 +- modules/portal/Gruntfile.js | 213 ++++++----- modules/portal/README.md | 36 +- .../app/views/groups/groupDetail.scala.html | 4 +- .../portal/app/views/groups/groups.scala.html | 4 +- modules/portal/app/views/login.scala.html | 5 +- modules/portal/app/views/main.scala.html | 78 +--- .../views/recordsets/recordSets.scala.html | 4 +- .../app/views/setOidcSession.scala.html | 3 +- .../portal/app/views/systemMessage.scala.html | 5 +- .../app/views/zones/zoneDetail.scala.html | 5 +- .../portal/app/views/zones/zones.scala.html | 4 +- modules/portal/karma.conf.js | 13 +- modules/portal/package.json | 18 +- modules/portal/prepare-portal.sh | 12 +- modules/portal/public/css/theme-overrides.css | 15 + modules/portal/public/css/vinyldns.css | 15 + modules/portal/public/js/custom.js | 40 +- .../lib/controllers/controller.zones.js | 2 +- .../dns-change/dns-change-new.controller.js | 2 - .../public/lib/dns-change/dns-change.spec.js | 2 +- .../lib/recordset/recordsets.controller.js | 2 - project/Dependencies.scala | 4 +- project/Resolvers.scala | 10 - project/assembly.sbt | 1 - project/plugins.sbt | 4 + project/protoc.sbt | 1 - quickstart/api/Dockerfile | 39 -- quickstart/api/Makefile | 50 --- quickstart/api/application.conf | 346 ------------------ quickstart/api/logback.xml | 32 -- quickstart/docker-compose.yml | 18 +- quickstart/portal/Dockerfile | 34 -- quickstart/portal/application.conf | 62 ---- quickstart/portal/application.ini | 3 - quickstart/quickstart-vinyldns.sh | 47 ++- test/api/functional/Makefile | 17 +- test/api/integration/Dockerfile | 9 +- test/api/integration/Makefile | 14 +- test/portal/functional/Makefile | 17 +- utils/admin/Dockerfile | 23 +- {quickstart/portal => utils/admin}/Makefile | 24 +- utils/admin/update-support-user.py | 6 +- utils/update-support-user.sh | 14 +- 62 files changed, 626 insertions(+), 1119 deletions(-) rename build/{assemble_api_jar.sh => assemble_api.sh} (73%) mode change 100755 => 100644 create mode 100644 build/assemble_portal.sh create mode 100644 build/docker/portal/logback.xml delete mode 100644 project/Resolvers.scala delete mode 100644 project/assembly.sbt delete mode 100644 project/protoc.sbt delete mode 100644 quickstart/api/Dockerfile delete mode 100644 quickstart/api/Makefile delete mode 100644 quickstart/api/application.conf delete mode 100644 quickstart/api/logback.xml delete mode 100644 quickstart/portal/Dockerfile delete mode 100644 quickstart/portal/application.conf delete mode 100644 quickstart/portal/application.ini rename {quickstart/portal => utils/admin}/Makefile (52%) diff --git a/.dockerignore b/.dockerignore index 995a18abd..c587c555b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,4 @@ **/_template img/ **/.env +modules/portal/node_modules/ diff --git a/.gitignore b/.gitignore index fc48f02ef..40aaed248 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,4 @@ quickstart/data **/.virtualenv **/.venv* **/*cache* -**/assembly/ +**/artifacts/ diff --git a/README.md b/README.md index 713244593..4068fd50f 100644 --- a/README.md +++ b/README.md @@ -14,25 +14,29 @@

# VinylDNS -VinylDNS is a vendor agnostic front-end for enabling self-service DNS and streamlining DNS operations. -VinylDNS manages millions of DNS records supporting thousands of engineers in production at [Comcast](http://www.comcast.com). -The platform provides fine-grained access controls, auditing of all changes, a self-service user interface, -secure RESTful API, and integration with infrastructure automation tools like Ansible and Terraform. -It is designed to integrate with your existing DNS infrastructure, and provides extensibility to fit your installation. + +VinylDNS is a vendor-agnostic front-end for enabling self-service DNS and streamlining DNS operations. VinylDNS manages +millions of DNS records supporting thousands of engineers in production at [Comcast](http://www.comcast.com). The +platform provides fine-grained access controls, auditing of all changes, a self-service user interface, secure RESTful +API, and integration with infrastructure automation tools like Ansible and Terraform. It is designed to integrate with +your existing DNS infrastructure, and provides extensibility to fit your installation. VinylDNS helps secure DNS management via: + - AWS Sig4 signing of all messages to ensure that the message that was sent was not altered in transit - Throttling of DNS updates to rate limit concurrent updates against your DNS systems - Encrypting user secrets and TSIG keys at rest and in-transit - Recording every change made to DNS records and zones Integration is simple with first-class language support including: + - Java - Python - Go - JavaScript ## Table of Contents + - [Quickstart](#quickstart) - [Code of Conduct](#code-of-conduct) - [Developer Guide](#developer-guide) @@ -42,10 +46,12 @@ Integration is simple with first-class language support including: - [Credits](#credits) ## Quickstart -Docker images for VinylDNS live on Docker Hub at . -To start up a local instance of VinylDNS on your machine with docker: -1. Ensure that you have [docker](https://docs.docker.com/install/) and [docker-compose](https://docs.docker.com/compose/install/) +Docker images for VinylDNS live on Docker Hub at . To start up a local instance of +VinylDNS on your machine with docker: + +1. Ensure that you have [docker](https://docs.docker.com/install/) + and [docker-compose](https://docs.docker.com/compose/install/) 1. Clone the repo: `git clone https://github.com/vinyldns/vinyldns.git` 1. Navigate to repo: `cd vinyldns` 1. Run `./quickstart/quickstart-vinyldns.sh`. This will start up the api at `localhost:9000` and the portal @@ -76,7 +82,8 @@ endpoint `http://localhost:9000` ### Verifying Your Changes -VinylDNS will synchronize with the DNS backend. For the Quickstart this should be running on port `19001` on `localhost`. +VinylDNS will synchronize with the DNS backend. For the Quickstart this should be running on port `19001` on `localhost` +. To verify your changes, you can use a DNS resolution utility like `dig` @@ -106,7 +113,7 @@ the record data we entered. This project, and everyone participating in it, are governed by the [VinylDNS Code Of Conduct](CODE_OF_CONDUCT.md). By participating, you agree to this Code. Please report any violations to the code of conduct to -vinyldns-core@googlegroups.com. +[vinyldns-core@googlegroups.com](mailto:vinyldns-core@googlegroups.com). ## Developer Guide @@ -118,9 +125,10 @@ See the [Contributing Guide](CONTRIBUTING.md). ## Contact -- If you have any security concerns please contact the maintainers directly vinyldns-core@googlegroups.com +- If you have any security concerns please contact the maintainers directly [vinyldns-core@googlegroups.com](mailto:vinyldns-core@googlegroups.com) ## Maintainers and Contributors + The current maintainers (people who can merge pull requests) are: - Ryan Emerle ([@remerle](https://github.com/remerle)) @@ -129,17 +137,25 @@ The current maintainers (people who can merge pull requests) are: See [AUTHORS.md](AUTHORS.md) for the full list of contributors to VinylDNS. -See [MAINTAINERS.md](MAINTAINERS.md) for documentation specific to maintainers +See [MAINTAINERS.md](MAINTAINERS.md) for documentation specific to maintainers ## Credits -VinylDNS would not be possible without the help of many other pieces of open source software. Thank you open source world! -Initial development of DynamoDBHelper done by [Roland Kuhn](https://github.com/rkuhn) from https://github.com/akka/akka-persistence-dynamodb/blob/8d7495821faef754d97759f0d3d35ed18fc17cc7/src/main/scala/akka/persistence/dynamodb/journal/DynamoDBHelper.scala +VinylDNS would not be possible without the help of many other pieces of open source software. Thank you open source +world! -Given the Apache 2.0 license of VinylDNS, we specifically want to call out the following libraries and their corresponding licenses shown below. -- [logback-classic](https://github.com/qos-ch/logback) - [Eclipse Public License 1.0](https://www.eclipse.org/legal/epl-v10.html) -- [logback-core](https://github.com/qos-ch/logback) - [Eclipse Public License 1.0](https://www.eclipse.org/legal/epl-v10.html) -- [h2 database](http://h2database.com) - [Mozilla Public License, version 2.0](https://www.mozilla.org/MPL/2.0/) -- [pureconfig](https://github.com/pureconfig/pureconfig) - [Mozilla Public License, version 2.0](https://www.mozilla.org/MPL/2.0/) -- [pureconfig-macros](https://github.com/pureconfig/pureconfig) - [Mozilla Public License, version 2.0](https://www.mozilla.org/MPL/2.0/) -- [junit](https://junit.org/junit4/) - [Eclipse Public License 1.0](https://www.eclipse.org/legal/epl-v10.html) +Given the Apache 2.0 license of VinylDNS, we specifically want to call out the following libraries and their +corresponding licenses shown below. + +- [logback-classic](https://github.com/qos-ch/logback) + - [Eclipse Public License 1.0](https://www.eclipse.org/legal/epl-v10.html) +- [logback-core](https://github.com/qos-ch/logback) + - [Eclipse Public License 1.0](https://www.eclipse.org/legal/epl-v10.html) +- [h2 database](http://h2database.com) + - [Mozilla Public License, version 2.0](https://www.mozilla.org/MPL/2.0/) +- [pureconfig](https://github.com/pureconfig/pureconfig) + - [Mozilla Public License, version 2.0](https://www.mozilla.org/MPL/2.0/) +- [pureconfig-macros](https://github.com/pureconfig/pureconfig) + - [Mozilla Public License, version 2.0](https://www.mozilla.org/MPL/2.0/) +- [junit](https://junit.org/junit4/) + - [Eclipse Public License 1.0](https://www.eclipse.org/legal/epl-v10.html) diff --git a/build.sbt b/build.sbt index 318c3ac1d..a1af2654b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,14 +1,12 @@ import CompilerOptions._ import Dependencies._ -import Resolvers._ import microsites._ import org.scalafmt.sbt.ScalafmtPlugin._ import scoverage.ScoverageKeys.{coverageFailOnMinimum, coverageMinimum} +import scala.language.postfixOps import scala.util.Try -resolvers ++= additionalResolvers - lazy val IntegrationTest = config("it").extend(Test) // settings that should be inherited by all projects @@ -18,6 +16,7 @@ lazy val sharedSettings = Seq( organizationName := "Comcast Cable Communications Management, LLC", startYear := Some(2018), licenses += ("Apache-2.0", new URL("https://www.apache.org/licenses/LICENSE-2.0.txt")), + maintainer := "VinylDNS Maintainers ", scalacOptions ++= scalacOptionsByV(scalaVersion.value), scalacOptions in (Compile, doc) += "-no-link-warnings", // Use wart remover to eliminate code badness @@ -64,7 +63,7 @@ lazy val apiSettings = Seq( ) lazy val apiAssemblySettings = Seq( - assemblyOutputPath in assembly := file("assembly/vinyldns.jar"), + assemblyOutputPath in assembly := file("artifacts/vinyldns-api.jar"), test in assembly := {}, mainClass in assembly := Some("vinyldns.api.Boot"), mainClass in reStart := Some("vinyldns.api.Boot"), @@ -79,45 +78,11 @@ lazy val apiAssemblySettings = Seq( } ) -lazy val noPublishSettings = Seq( - publish := {}, - publishLocal := {}, - publishArtifact := false -) - -lazy val apiPublishSettings = Seq( - publishArtifact := false, - publishLocal := (publishLocal in Docker).value, - publish := (publish in Docker).value -) - -lazy val portalPublishSettings = Seq( - publishArtifact := false, - publishLocal := (publishLocal in Docker).value, - publish := (publish in Docker).value, - // for sbt-native-packager (docker) to exclude local.conf - mappings in Universal ~= (_.filterNot { - case (file, _) => file.getName.equals("local.conf") - }), - // for local.conf to be excluded in jars - mappings in (Compile, packageBin) ~= (_.filterNot { - case (file, _) => file.getName.equals("local.conf") - }) -) - -lazy val pbSettings = Seq( - PB.targets in Compile := Seq( - PB.gens.java("2.6.1") -> (sourceManaged in Compile).value - ), - PB.protocVersion := "-v261" -) - lazy val allApiSettings = Revolver.settings ++ Defaults.itSettings ++ apiSettings ++ sharedSettings ++ apiAssemblySettings ++ - testSettings ++ - apiPublishSettings + testSettings lazy val api = (project in file("modules/api")) .enablePlugins(JavaAppPackaging, AutomateHeaderPlugin) @@ -146,8 +111,10 @@ lazy val coreBuildSettings = Seq( name := "core", // do not use unused params as NoOpCrypto ignores its constructor, we should provide a way // to write a crypto plugin so that we fall back to a noarg constructor - scalacOptions ++= scalacOptionsByV(scalaVersion.value).filterNot(_ == "-Ywarn-unused:params") -) ++ pbSettings + scalacOptions ++= scalacOptionsByV(scalaVersion.value).filterNot(_ == "-Ywarn-unused:params"), + PB.targets in Compile := Seq(PB.gens.java("2.6.1") -> (sourceManaged in Compile).value), + PB.protocVersion := "-v261" +) lazy val corePublishSettings = Seq( publishMavenStyle := true, @@ -156,7 +123,6 @@ lazy val corePublishSettings = Seq( false }, autoAPIMappings := true, - publish in Docker := {}, mainClass := None, homepage := Some(url("https://vinyldns.io")), scmInfo := Some( @@ -233,42 +199,47 @@ val checkJsHeaders = val createJsHeaders = TaskKey[Unit]("createJsHeaders", "Runs script to prepend APL 2.0 license headers to files") +lazy val portalSettings = Seq( + libraryDependencies ++= portalDependencies, + routesGenerator := InjectedRoutesGenerator, + coverageExcludedPackages := ";views.html.*;router.*;controllers\\.javascript.*;.*Reverse.*", + javaOptions in Test += "-Dconfig.file=conf/application-test.conf", + // ads the version when working locally with sbt run + PlayKeys.devSettings += "vinyldns.base-version" -> (version in ThisBuild).value, + // adds an extra classpath to the portal loading so we can externalize jars, make sure to create the lib_extra + // directory and lay down any dependencies that are required when deploying + scriptClasspath in bashScriptDefines ~= (cp => cp :+ "lib_extra/*"), + mainClass in reStart := None, + // we need to filter out unused for the portal as the play framework needs a lot of unused things + scalacOptions ~= { opts => + opts.filterNot(p => p.contains("unused")) + }, + // runs our prepare portal process + preparePortal := { + import scala.sys.process._ + "./modules/portal/prepare-portal.sh" ! + }, + checkJsHeaders := { + import scala.sys.process._ + "./utils/add-license-headers.sh -d=modules/portal/public/lib -f=js -c" ! + }, + createJsHeaders := { + import scala.sys.process._ + "./utils/add-license-headers.sh -d=modules/portal/public/lib -f=js" ! + }, + + // Change the path of the output to artifacts/vinyldns-portal.zip + target in Universal := file("artifacts/"), + packageName in Universal := "vinyldns-portal" +) + lazy val portal = (project in file("modules/portal")) .enablePlugins(PlayScala, AutomateHeaderPlugin) .settings(sharedSettings) .settings(testSettings) - .settings(portalPublishSettings) + .settings(portalSettings) .settings( name := "portal", - libraryDependencies ++= portalDependencies, - routesGenerator := InjectedRoutesGenerator, - coverageExcludedPackages := ";views.html.*;router.*;controllers\\.javascript.*;.*Reverse.*", - javaOptions in Test += "-Dconfig.file=conf/application-test.conf", - // ads the version when working locally with sbt run - PlayKeys.devSettings += "vinyldns.base-version" -> (version in ThisBuild).value, - // adds an extra classpath to the portal loading so we can externalize jars, make sure to create the lib_extra - // directory and lay down any dependencies that are required when deploying - scriptClasspath in bashScriptDefines ~= (cp => cp :+ "lib_extra/*"), - mainClass in reStart := None, - // we need to filter out unused for the portal as the play framework needs a lot of unused things - scalacOptions ~= { opts => - opts.filterNot(p => p.contains("unused")) - }, - // runs our prepare portal process - preparePortal := { - import scala.sys.process._ - "./modules/portal/prepare-portal.sh" ! - }, - checkJsHeaders := { - import scala.sys.process._ - "./utils/add-license-headers.sh -d=modules/portal/public/lib -f=js -c" ! - }, - createJsHeaders := { - import scala.sys.process._ - "./utils/add-license-headers.sh -d=modules/portal/public/lib -f=js" ! - }, - // change the name of the output to portal.zip - packageName in Universal := "portal" ) .dependsOn(mysql) diff --git a/build/assemble_api_jar.sh b/build/assemble_api.sh old mode 100755 new mode 100644 similarity index 73% rename from build/assemble_api_jar.sh rename to build/assemble_api.sh index 5e7c6f34b..ab9970ce6 --- a/build/assemble_api_jar.sh +++ b/build/assemble_api.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # -# This script will build the vinyldns.jar file using Docker. The file will -# be placed in the configured location (currently `assembly/` off of the root) +# This script will build the vinyldns-api.jar file using Docker. The file will +# be placed in the configured location (currently `artifacts/` off of the root) # set -euo pipefail @@ -11,7 +11,7 @@ DIR=$( ) usage() { - echo "USAGE: assemble_jar.sh [options]" + echo "USAGE: assemble_api.sh [options]" echo -e "\t-n, --no-clean do no perform a clean before assembling the jar" echo -e "\t-u, --update update the underlying docker image" } @@ -37,6 +37,7 @@ done if ! [[ $SKIP_CLEAN -eq 1 ]]; then "${DIR}/deep_clean.sh" + rm "${DIR}/../artifacts/vinyldns-api.jar" &> /dev/null || true fi if [[ $UPDATE_DOCKER -eq 1 ]]; then @@ -44,5 +45,5 @@ if [[ $UPDATE_DOCKER -eq 1 ]]; then docker pull vinyldns/build:base-test-integration fi -echo "Building VinylDNS API jar file" +echo "Building VinylDNS API artifact" docker run -i --rm -e RUN_SERVICES=none -v "${DIR}/..:/build" vinyldns/build:base-test-integration -- sbt 'api/assembly' diff --git a/build/assemble_portal.sh b/build/assemble_portal.sh new file mode 100644 index 000000000..ad10103c0 --- /dev/null +++ b/build/assemble_portal.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# This script will build the vinyldns-portal.zip file using Docker. The file will +# be placed in the configured location (currently `artifacts/` off of the root) +# +set -euo pipefail + +DIR=$( + cd "$(dirname "$0")" + pwd -P +) + +usage() { + echo "USAGE: assemble_portal.sh [options]" + echo -e "\t-n, --no-clean do no perform a clean before assembling the jar" + echo -e "\t-u, --update update the underlying docker image" +} + +SKIP_CLEAN=0 +UPDATE_DOCKER=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --no-clean | -n) + SKIP_CLEAN=1 + shift + ;; + --update | -u) + UPDATE_DOCKER=1 + shift + ;; + *) + usage + exit 1 + ;; + esac +done + +if ! [[ $SKIP_CLEAN -eq 1 ]]; then + "${DIR}/deep_clean.sh" + rm "${DIR}/../artifacts/vinyldns-portal.zip" &> /dev/null || true + rm -rf "${DIR}/../artifacts/scripts" &> /dev/null || true +fi + +if [[ $UPDATE_DOCKER -eq 1 ]]; then + echo "Pulling latest version of 'vinyldns/build:base-test-integration'" + docker pull vinyldns/build:base-test-integration +fi + +echo "Building VinylDNS Portal artifact" +docker run -i --rm -e RUN_SERVICES=none -v "${DIR}/..:/build" vinyldns/build:base-test-integration -- sbt 'portal/dist' diff --git a/build/docker/README.md b/build/docker/README.md index b913d5d62..e195766f3 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -60,9 +60,9 @@ The default build for vinyldns api assumes an **ALL MYSQL** installation. it is set as part of the container build **Volumes** -- `/opt/docker/conf/` - if you need to have your own application config file. This is **MANDATORY** for +- `/opt/vinyldns/conf/` - if you need to have your own application config file. This is **MANDATORY** for any production environments. Typically, you will add your own `application.conf` file in here with your settings. -- `/opt/docker/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. +- `/opt/vinyldns/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. Rarely used, but if you want to bring your own message queue or database you can put the `jar` files there ### vinyldns/portal @@ -74,7 +74,7 @@ The default build for vinyldns portal assumes an **ALL MYSQL** installation. it is set as part of the container build **Volumes** -- `/opt/docker/conf/` - if you need to have your own application config file. This is **MANDATORY** for +- `/opt/vinyldns/conf/` - if you need to have your own application config file. This is **MANDATORY** for any production environments. Typically, you will add your own `application.conf` file in here with your settings. -- `/opt/docker/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. +- `/opt/vinyldns/lib_extra/` - if you need to have additional jar files available to your VinylDNS instance. Rarely used, but if you want to bring your own message queue or database you can put the `jar` files there diff --git a/build/docker/api/Dockerfile b/build/docker/api/Dockerfile index 3a228f48e..89ea8d028 100644 --- a/build/docker/api/Dockerfile +++ b/build/docker/api/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /build ## Run the build if we don't already have a vinyldns.jar RUN mkdir -p /opt/vinyldns/conf && \ - if [ -f assembly/vinyldns.jar ]; then cp assembly/vinyldns.jar /opt/vinyldns/; fi && \ + if [ -f artifacts/vinyldns-api.jar ]; then cp artifacts/vinyldns-api.jar /opt/vinyldns/; fi && \ if [ ! -f /opt/vinyldns/vinyldns.jar ]; then \ env SBT_OPTS="-XX:+UseConcMarkSweepGC -Xmx4G -Xms1G" \ sbt -Dbuild.scalafmtOnCompile=false -Dbuild.lintOnCompile=fase ";project api;coverageOff;assembly" \ @@ -16,21 +16,24 @@ FROM adoptopenjdk/openjdk11:jdk-11.0.8_10-alpine ARG DOCKER_FILE_PATH ARG VINYLDNS_VERSION -RUN apk add --update --no-cache bash && mkdir -p /opt/vinyldns/lib_extra +RUN test -n "VINYLDNS_VERSION" || (echo "VINYLDNS_VERSION not set" && false) && \ + test -n "DOCKER_FILE_PATH" || (echo "DOCKER_FILE_PATH not set" && false) && \ + apk add --update --no-cache bash && \ + mkdir -p /opt/vinyldns/lib_extra && \ + echo "${VINYLDNS_VERSION}" > /opt/vinyldns/version COPY --from=base-build /opt/vinyldns /opt/vinyldns COPY ${DOCKER_FILE_PATH}/application.conf /opt/vinyldns/conf COPY ${DOCKER_FILE_PATH}/logback.xml /opt/vinyldns/conf -RUN echo "${VINYLDNS_VERSION}" > /opt/vinyldns/conf/version - # Mount the volume for config file and lib extras VOLUME ["/opt/vinyldns/lib_extra/", "/opt/vinyldns/conf/"] EXPOSE 9000 -ENTRYPOINT ["/bin/bash", "-c", "java -Dconfig.file=/opt/vinyldns/conf/application.conf \ +ENV JVM_OPTS="" +ENTRYPOINT ["/bin/bash", "-c", "java ${JVM_OPTS} -Dconfig.file=/opt/vinyldns/conf/application.conf \ -Dlogback.configurationFile=/opt/vinyldns/conf/logback.xml \ - -Dvinyldns.version=$(cat /opt/vinyldns/conf/version) \ + -Dvinyldns.version=$(cat /opt/vinyldns/version) \ -cp /opt/vinyldns/lib_extra/* \ -jar /opt/vinyldns/vinyldns.jar" ] diff --git a/build/docker/api/Makefile b/build/docker/api/Makefile index d0eb622c2..f9ba5fd59 100644 --- a/build/docker/api/Makefile +++ b/build/docker/api/Makefile @@ -10,7 +10,7 @@ ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) endif # Extract arguments for `make run` -EXTRACT_ARGS=true +EXTRACT_ARGS=false ifeq (run,$(firstword $(MAKECMDGOALS))) EXTRACT_ARGS=true endif @@ -18,8 +18,8 @@ ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif -ifdef $(WITH_ARGS) - ARG_SEPARATOR=-- +ifneq ($(WITH_ARGS),) + ARG_SEPARATOR=-- endif %: @@ -40,7 +40,7 @@ build: run: @set -euo pipefail docker network create --driver bridge vinyldns_net &> /dev/null || true - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -i $${USE_TTY} --rm --env-file "$(ROOT_DIR)/../.env" --network vinyldns_net $(DOCKER_PARAMS) -v "$$(pwd)/:/opt/vinyldns/conf/" -p 9000:9000 $(IMAGE_NAME):$(IMAGE_TAG) $(ARG_SEPARATOR) $(WITH_ARGS) publish: build diff --git a/build/docker/api/application.conf b/build/docker/api/application.conf index 2e9be4d49..201018765 100644 --- a/build/docker/api/application.conf +++ b/build/docker/api/application.conf @@ -1,5 +1,4 @@ vinyldns { - base-version = "0.0.0-local-dev" version = ${vinyldns.base-version} # default to the base version if not overridden version = ${?VINYLDNS_VERSION} # override the base version via env var @@ -125,6 +124,7 @@ vinyldns { class-name = ${?EMAIL_CLASS_NAME} settings = { from = "VinylDNS " + from = ${?EMAIL_FROM} } } @@ -161,12 +161,13 @@ vinyldns { "ns1.parent.com4." ] + # Note: This MUST match the Portal or strange errors will ensue, NoOpCrypto should not be used for production crypto { type = "vinyldns.core.crypto.NoOpCrypto" + type = ${?CRYPTO_TYPE} + secret = ${?CRYPTO_SECRET} } - data-stores = ["mysql"] - mysql { settings { # JDBC Settings, these are all values in scalikejdbc-config, not our own @@ -184,49 +185,11 @@ vinyldns { user = ${?JDBC_USER} password = "" password = ${?JDBC_PASSWORD} - # see https://github.com/brettwooldridge/HikariCP - connection-timeout-millis = 1000 - idle-timeout = 10000 - max-lifetime = 600000 - maximum-pool-size = 20 - minimum-idle = 20 - register-mbeans = true - } - # Repositories that use this data store are listed here - repositories { - zone { - # no additional settings for now - } - batch-change { - # no additional settings for now - } - user { - - } - record-set { - - } - group { - - } - membership { - - } - group-change { - - } - zone-change { - - } - record-change { - - } } } backends = [] - # FQDNs / IPs that cannot be modified via VinylDNS # regex-list used for all record types except PTR # ip-list used exclusively for PTR records @@ -343,3 +306,7 @@ akka.http { illegal-header-warnings = on } } + +# You can provide configuration overrides via local.conf if you don't want to replace everything in +# this configuration file +include "local.conf" diff --git a/build/docker/portal/Dockerfile b/build/docker/portal/Dockerfile index d51332172..e1194a574 100644 --- a/build/docker/portal/Dockerfile +++ b/build/docker/portal/Dockerfile @@ -1,37 +1,47 @@ -FROM vinyldns/build:base-build-portal as builder +FROM vinyldns/build:base-build-portal as base-build ARG VINYLDNS_VERSION +COPY . /build +WORKDIR /build -COPY . /vinyldns - -WORKDIR /vinyldns -RUN cp /build/node_modules.tar.xz /vinyldns/modules/portal && \ - cd /vinyldns/modules/portal && tar Jxf node_modules.tar.xz && \ - cd /vinyldns - -RUN sbt "set version in ThisBuild := \"${VINYLDNS_VERSION}\"; project portal; preparePortal" -RUN sbt "set version in ThisBuild := \"${VINYLDNS_VERSION}\"; project portal; universal:packageZipTarball" +RUN mkdir -p /opt/vinyldns/conf && \ + if [ -f artifacts/vinyldns-portal.zip ]; then \ + unzip artifacts/vinyldns-portal.zip -d /opt/vinyldns && \ + mv /opt/vinyldns/vinyldns-portal/{lib,share,conf} /opt/vinyldns && \ + rm -rf /opt/vinyldns/vinyldns-portal*; \ + fi && \ + if [ ! -f /opt/vinyldns/lib/vinyldns.portal*.jar ]; then \ + cp /build/node_modules.tar.xz /build/modules/portal && \ + cd /build/modules/portal && tar Jxf node_modules.tar.xz && \ + cd /build && \ + modules/portal/prepare-portal.sh && \ + sbt "set version in ThisBuild := \"${VINYLDNS_VERSION}\"; project portal; dist" && \ + unzip artifacts/vinyldns-portal.zip -d /opt/vinyldns && \ + mv /opt/vinyldns/vinyldns-portal/{lib,share,conf} /opt/vinyldns && \ + rm -rf /opt/vinyldns/vinyldns-portal*; \ + fi FROM adoptopenjdk/openjdk11:jdk-11.0.8_10-alpine ARG DOCKER_FILE_PATH ARG VINYLDNS_VERSION -RUN test -n "VINYLDNS_VERSION" || (echo "VINYLDNS_VERSION not set" && false) -COPY --from=builder /vinyldns/modules/portal/target/universal/portal.tgz / - -RUN apk add --update --no-cache bash && \ +RUN test -n "VINYLDNS_VERSION" || (echo "VINYLDNS_VERSION not set" && false) && \ + test -n "DOCKER_FILE_PATH" || (echo "DOCKER_FILE_PATH not set" && false) && \ + apk add --update --no-cache bash && \ mkdir -p /opt/vinyldns/lib_extra && \ - tar -xzf /portal.tgz && \ - rm /portal.tgz && \ - mv /portal/* /opt/vinyldns && \ - echo "${VINYLDNS_VERSION}" > /opt/vinyldns/conf/version + echo "${VINYLDNS_VERSION}" > /opt/vinyldns/version + +COPY --from=base-build /opt/vinyldns /opt/vinyldns +COPY ${DOCKER_FILE_PATH}/application.conf /opt/vinyldns/conf +COPY ${DOCKER_FILE_PATH}/logback.xml /opt/vinyldns/conf -COPY ${DOCKER_FILE_PATH}/application.conf /opt/vinyldns/conf/ VOLUME ["/opt/vinyldns/lib_extra/", "/opt/vinyldns/conf/"] EXPOSE 9001 -ENTRYPOINT ["/bin/bash","-c", "java -Dvinyldns.version=$(cat /opt/vinyldns/conf/version) \ - -Dconfig.file=/opt/vinyldns/conf/application.conf \ - -cp /opt/vinyldns/conf:/opt/vinyldns/lib/*:/opt/vinyldns/lib_extra/* \ - play.core.server.ProdServerStart"] +ENV JVM_OPTS="" +ENTRYPOINT ["/bin/bash","-c", "java ${JVM_OPTS} -Dvinyldns.version=$(cat /opt/vinyldns/version) \ + -Dlogback.configurationFile=/opt/vinyldns/conf/logback.xml \ + -Dconfig.file=/opt/vinyldns/conf/application.conf \ + -cp /opt/vinyldns/conf:/opt/vinyldns/lib/*:/opt/vinyldns/lib_extra/* \ + play.core.server.ProdServerStart"] diff --git a/build/docker/portal/Makefile b/build/docker/portal/Makefile index eaad9a558..be2b8b2a5 100644 --- a/build/docker/portal/Makefile +++ b/build/docker/portal/Makefile @@ -10,7 +10,7 @@ ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) endif # Extract arguments for `make run` -EXTRACT_ARGS=true +EXTRACT_ARGS=false ifeq (run,$(firstword $(MAKECMDGOALS))) EXTRACT_ARGS=true endif @@ -18,8 +18,8 @@ ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif -ifdef $(WITH_ARGS) - ARG_SEPARATOR=-- +ifneq ($(WITH_ARGS),) + ARG_SEPARATOR=-- endif %: @@ -44,8 +44,8 @@ build: run: @set -euo pipefail docker network create --driver bridge vinyldns_net &> /dev/null || true - test -t 1 && USE_TTY="-t" - docker run -i $${USE_TTY} --rm --network vinyldns_net --env-file "$(ROOT_DIR)/../.env" $(DOCKER_PARAMS) -v "$$(pwd)/application.conf:/opt/vinyldns/conf/application.conf" -p 9001:9001 $(IMAGE_NAME):$(IMAGE_TAG) $(ARG_SEPARATOR) $(WITH_ARGS) + USE_TTY="" && test -t 1 && USE_TTY="-t" + docker run -i $${USE_TTY} --rm --network vinyldns_net --env-file "$(ROOT_DIR)/../.env" $(DOCKER_PARAMS) -v "$$(pwd)/application.conf:/opt/vinyldns/conf/application.conf" -v "$$(pwd)/logback.xml:/opt/vinyldns/conf/logback.xml" -p 9001:9001 $(IMAGE_NAME):$(IMAGE_TAG) $(ARG_SEPARATOR) $(WITH_ARGS) publish: build @set -euo pipefail diff --git a/build/docker/portal/application.conf b/build/docker/portal/application.conf index 40ae2c83a..35c7809e1 100644 --- a/build/docker/portal/application.conf +++ b/build/docker/portal/application.conf @@ -18,7 +18,9 @@ LDAP { ] context { initialContextFactory = "com.sun.jndi.ldap.LdapCtxFactory" + initialContextFactory = ${?LDAP_INITIAL_CONTEXT_CLASS} securityAuthentication = "simple" + securityAuthentication = ${?LDAP_SECURITY_AUTH} # Note: The following assumes a purely docker setup, using container_name = vinyldns-ldap providerUrl = "ldap://vinyldns-ldap:19004" @@ -29,13 +31,17 @@ LDAP { # company for example) user-sync { enabled = false + enabled = ${?USER_SYNC_ENABLED} hours-polling-interval = 1 + hours-polling-interval = ${?USER_SYNC_POLL_INTERVAL} } } -# Note: This MUST match the API or strange errors will ensure, NoCrypto should not be used for production +# Note: This MUST match the API or strange errors will ensue, NoOpCrypto should not be used for production crypto { type = "vinyldns.core.crypto.NoOpCrypto" + type = ${?CRYPTO_TYPE} + secret = ${?CRYPTO_SECRET} } http.port = 9001 @@ -43,20 +49,14 @@ http.port = ${?PORTAL_PORT} data-stores = ["mysql"] -# Note: The default mysql settings assume a local docker compose setup with mysql named vinyldns-mysql -# follow the configuration guide to point to your mysql -# Only 3 repositories are needed for portal: user, task, user-change -mysql { - repositories { - user { - } - task { - } - user-change { - } - } -} +# Must be true to manage shared zones through the portal +shared-display-enabled = true +shared-display-enabled = ${?SHARED_ZONES_ENABLED} # You generate this yourself following https://www.playframework.com/documentation/2.7.x/ApplicationSecret -play.http.secret.key = "rpkTGtoJvLIdIV?WU=0@yW^x:pcEGyAt`^p/P3G0fpbj9:uDnD@caSjCDqA0@tB=" +play.http.secret.key = "changeme" play.http.secret.key = ${?PLAY_HTTP_SECRET_KEY} + +# You can provide configuration overrides via local.conf if you don't want to replace everything in +# this configuration file +include "local.conf" diff --git a/build/docker/portal/logback.xml b/build/docker/portal/logback.xml new file mode 100644 index 000000000..f8a08e496 --- /dev/null +++ b/build/docker/portal/logback.xml @@ -0,0 +1,21 @@ + + + + + + + %d{"yyyy-MM-dd HH:mm:ss,SSS"} %coloredLevel - %logger - %message%n%xException + + + + + + + + + + + diff --git a/build/run_all_tests.sh b/build/run_all_tests.sh index ea9312107..36cdc2c62 100755 --- a/build/run_all_tests.sh +++ b/build/run_all_tests.sh @@ -4,8 +4,8 @@ DIR=$(cd -P -- "$(dirname -- "$0")" && pwd -P) source "${DIR}/../utils/includes/terminal_colors.sh" -if [ ! -d "${DIR}/../assembly" ] || [ ! -f "${DIR}/../assembly/vinyldns.jar" ]; then - echo -e "${F_YELLOW}Warning:${F_RESET} you might want to run 'build/assemble_api_jar.sh' first to improve performance" +if [ ! -d "${DIR}/../artifacts" ] || [ ! -f "${DIR}/../artifacts/vinyldns-api.jar" ]; then + echo -e "${F_YELLOW}Warning:${F_RESET} you might want to run 'build/assemble_api.sh' first to improve performance" fi echo "Running unit and integration tests..." diff --git a/modules/docs/Makefile b/modules/docs/Makefile index 05bde5c82..0076476ad 100644 --- a/modules/docs/Makefile +++ b/modules/docs/Makefile @@ -30,5 +30,5 @@ all: run run: @set -euo pipefail cd ../.. - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -i $${USE_TTY} --rm -p "4000:4000" -v "$$(pwd):/build" vinyldns/build:base-build-docs /bin/bash diff --git a/modules/docs/src/main/mdoc/operator/setup-api.md b/modules/docs/src/main/mdoc/operator/setup-api.md index a88cc5ce7..722650be1 100644 --- a/modules/docs/src/main/mdoc/operator/setup-api.md +++ b/modules/docs/src/main/mdoc/operator/setup-api.md @@ -29,9 +29,9 @@ purposes. ## Volume Mounts The API exposes volumes that allow the user to customize the runtime. Those mounts include: -* `/opt/docker/lib_extra` - place here additional jar files that need to be loaded into the classpath when the application starts up. +* `/opt/vinyldns/lib_extra` - place here additional jar files that need to be loaded into the classpath when the application starts up. This is used for "plugins" that are proprietary or not part of the standard build. All jar files here will be placed on the class path. -* `/opt/docker/conf` - place an `application.conf` file here with your own custom settings. Once you have your config created, +* `/opt/vinyldns/conf` - place an `application.conf` file here with your own custom settings. Once you have your config created, place here. ## Ports diff --git a/modules/docs/src/main/mdoc/operator/setup-portal.md b/modules/docs/src/main/mdoc/operator/setup-portal.md index 527d75376..f16369c80 100644 --- a/modules/docs/src/main/mdoc/operator/setup-portal.md +++ b/modules/docs/src/main/mdoc/operator/setup-portal.md @@ -1,11 +1,11 @@ --- -layout: docs -title: "Setup the Portal Server" +layout: docs title: "Setup the Portal Server" section: "operator_menu" --- # Setup the Portal Server -The Portal Server is the web UI for VinylDNS. To setup the Portal server, follow these steps: + +The Portal Server is the web UI for VinylDNS. To setup the Portal server, follow these steps: 1. [Setup API Server](setup-api.html) 1. [Setup LDAP](setup-ldap.html) @@ -16,21 +16,32 @@ Once you have you pre-requisites ready, review the [Portal Configuration Guide]( your configuration file. # Using the Portal Docker Image + The Portal server is provided as a [VinylDNS Portal Image](https://hub.docker.com/r/vinyldns/portal/). The API server is _stateless_, allowing you to run multiple instances in multiple data centers for high-availability purposes. ## Volume mounts -* `/opt/docker/lib_extra` - place here additional jar files that need to be loaded into the classpath when the application starts up. -This is used for "plugins" that are proprietary or not part of the standard build. All jar files here will be placed on the class path. -* `/opt/docker/conf/application.conf` - to override default configuration settings. Follow the [Portal Configuration Guide](config-portal.html) -* `/opt/docker/conf/application.ini` - to pass additional JVM options -* `/opt/docker/conf/trustStore.jks` - to make available a custom trustStore, which has to be set in `/opt/docker/conf/application.ini` as `-Djavax.net.ssl.trustStore=/opt/docker/conf/trustStore.jks` + +* `/opt/vinyldns/lib_extra` - place here additional jar files that need to be loaded into the classpath when the + application starts up. This is used for "plugins" that are proprietary or not part of the standard build. All jar + files here will be placed on the class path. +* `/opt/vinyldns/conf/application.conf` - to override default configuration settings. Follow + the [Portal Configuration Guide](config-portal.html) ## Configuring a custom Java trustStore -To add a custom Java trustStore for LDAP certs, add the trustStore to `/opt/docker/conf/trustStore.jks`. - Then add `-Djavax.net.ssl.trustStore=/opt/docker/conf/trustStore.jks` to `/opt/docker/conf/application.ini`. + +To add a custom Java trustStore for LDAP certs, add the trustStore to `/opt/vinyldns/conf/trustStore.jks`. Then +add `-Djavax.net.ssl.trustStore=/opt/vinyldns/conf/trustStore.jks` to the `JVM_OPTS` environment variable for the +container. + +Example: + +```text +docker run -e JVM_OPTS="-Djavax.net.ssl.trustStore=/opt/vinyldns/conf/trustStore.jks" ... +``` ## Additional JVM parameters -Additional JVM parameters can be added to `/opt/docker/conf/application.ini` + +Additional JVM parameters can be added to the `JVM_OPTS` environment variable diff --git a/modules/portal/.gitignore b/modules/portal/.gitignore index d788b9954..843786a95 100644 --- a/modules/portal/.gitignore +++ b/modules/portal/.gitignore @@ -4,13 +4,18 @@ __pycache__ /func_test/.virtualenv node_modules/ public/bower_components/ -public/javascripts/ -public/stylesheets/ public/test_frameworks/ -public/custom/views.vinyl.js package-lock.json release.version private -public/gentelella .bloop .metals + +public/js/* +!public/js/custom.js + +public/css/* +!public/css/theme-overrides.css +!public/css/vinyldns.css + +public/fonts diff --git a/modules/portal/Gruntfile.js b/modules/portal/Gruntfile.js index eb3ef53e8..b541d112e 100644 --- a/modules/portal/Gruntfile.js +++ b/modules/portal/Gruntfile.js @@ -1,88 +1,133 @@ -/*global module:false*/ -module.exports = function (grunt) { +module.exports = function(grunt) { - // Project configuration. - grunt.initConfig({ - // Metadata. - pkg: grunt.file.readJSON('package.json'), - copy: { - main: { - files: [ - // includes files within path and its sub-directories - { expand: true, flatten: true, src: ['node_modules/jquery/dist/jquery.min.js'], dest: 'public/javascripts' }, - { expand: true, flatten: true, src: ['node_modules/angular/angular.min.js'], dest: 'public/javascripts' }, - { expand: true, flatten: true, src: ['node_modules/angular-animate/angular-animate.min.js'], dest: 'public/javascripts' }, - { expand: true, flatten: true, src: ['node_modules/angular-bootstrap/ui-bootstrap.min.js'], dest: 'public/javascripts' }, - { expand: true, flatten: true, src: ['node_modules/bootstrap/dist/js/bootstrap.min.js'], dest: 'public/javascripts' }, - { expand: true, flatten: true, src: ['node_modules/angular-ui-router/release/angular-ui-router.min.js'], dest: 'public/javascripts' }, - { expand: true, flatten: true, src: ['node_modules/bootstrap/dist/css/bootstrap.min.css'], dest: 'public/stylesheets' }, - { expand: true, flatten: true, src: ['node_modules/font-awesome/css/font-awesome.min.css'], dest: 'public/stylesheets' }, - { expand: true, flatten: true, src: ['node_modules/moment/min/moment.min.js'], dest: 'public/javascripts' }, - { expand: true, cwd: 'node_modules/gentelella', dest: 'public/gentelella', src: '**'}, - { expand: true, flatten: true, src: ['public/custom/**/*.js', '!public/custom/**/*.spec.js'], dest: 'public/javascripts' }, - { expand: true, flatten: true, src: ['public/custom/**/*.css'], dest: 'public/stylesheets' } - ] - }, - unit: { - files: [ - { expand: true, flatten: true, src: ['node_modules/angular-mocks/angular-mocks.js'], dest: 'public/test_frameworks' }, - { expand: true, flatten: true, src: ['node_modules/jasmine-jquery/lib/jasmine-jquery.js'], dest: 'public/test_frameworks' }, - ] - } - }, - injector: { - local_dependencies: { - files: { - 'app/views/main.scala.html': [ - 'public/javascripts/moment.min.js', - 'public/gentelella/vendors/jquery/dist/jquery.min.js', - 'public/gentelella/vendors/bootstrap/dist/js/bootstrap.min.js', - 'public/gentelella/vendors/bootstrap-daterangepicker/daterangepicker.js', - 'public/javascripts/ui-bootstrap.min.js', - 'public/javascripts/angular.min.js', - 'public/lib/**/*.module.js', - 'public/lib/**/*.js', - 'public/app.js', - 'public/gentelella/build/js/custom.js', - 'public/js/custom.js', - '!public/lib/**/*.spec.js' - ] - } - } - }, - karma: { - unit: { - configFile: 'karma.conf.js' - } - }, - clean: { - js: ['public/javascripts/*'], - css: ['public/stylesheets/*'], - unit: ['public/test_frameworks/*'], - gentelella: ['public/gentelella/*'] - }, - ngtemplates: { - 'views.vinyl': { - src: 'public/custom/**/*.html', - dest: 'public/custom/views.vinyl.js', - options: { - standalone: true, - quotes: 'single', - url: function (url) { return url.replace('public/custom/', ''); } - } - } - } - }); + const license = `/* +* Copyright 2018 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the \\"License\\"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an \\"AS IS\\" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/`; - grunt.loadNpmTasks('grunt-contrib-copy'); - grunt.loadNpmTasks('grunt-injector'); - grunt.loadNpmTasks('grunt-contrib-clean'); - grunt.loadNpmTasks('grunt-angular-templates'); - grunt.loadNpmTasks('grunt-karma'); - grunt.loadNpmTasks('grunt-mocha-phantomjs'); + grunt.loadNpmTasks('grunt-contrib-copy'); + grunt.loadNpmTasks('grunt-contrib-clean'); + grunt.loadNpmTasks('grunt-karma'); + grunt.loadNpmTasks('grunt-mocha-phantomjs'); + grunt.loadNpmTasks('grunt-contrib-concat'); - // Default task - grunt.registerTask('default', ['clean', 'ngtemplates', 'copy:main', 'injector']); - // Unit tests - grunt.registerTask('unit', ['default', 'ngtemplates', 'copy:unit', 'karma:unit', 'clean:unit']); + // Project configuration. + grunt.initConfig({ + pkg: grunt.file.readJSON('package.json'), + + copy: { + // Collect all of the javascript and CSS files that we need (we'll combine them later) + main: { + files: [ + {expand: true, flatten: true, src: ['node_modules/angular/angular.min.js'], dest: 'public/js'}, + {expand: true, flatten: true, src: ['node_modules/bootstrap/dist/js/bootstrap.min.js'], dest: 'public/js'}, + {expand: true, flatten: true, src: ['node_modules/jquery/dist/jquery.min.js'], dest: 'public/js'}, + {expand: true, flatten: true, src: ['node_modules/moment/min/moment.min.js'], dest: 'public/js'}, + + {expand: true, flatten: true, src: ['node_modules/bootstrap/dist/css/bootstrap.min.css'], dest: 'public/css'}, + {expand: true, flatten: true, src: ['node_modules/font-awesome/css/font-awesome.min.css'], dest: 'public/css'}, + + // We're picking just the resources we need from the gentelella UI framework and temporarily storing them in mapped/ui/ + {expand: true, flatten: true, cwd: 'node_modules/gentelella', dest: 'mapped/ui', src: '**/jquery.{smartWizard,mCustomScrollbar.concat.min,dataTables.min,mousewheel.min}.js'}, + {expand: true, flatten: true, cwd: 'node_modules/gentelella', dest: 'mapped/ui', src: '**/bootstrap-daterangepicker/daterangepicker.js'}, + {expand: true, flatten: true, cwd: 'node_modules/gentelella', dest: 'mapped/ui', src: '**/build/css/custom.min.css'}, + {expand: true, flatten: true, cwd: 'node_modules/gentelella', dest: 'mapped/ui', src: '**/{daterangepicker,animate.min}.css'}, + {expand: true, flatten: true, cwd: 'node_modules/gentelella', dest: 'public/fonts', src: '**/fonts/*.{woff,woff2,ttf}'}, + + {expand: true, flatten: true, src: ['public/custom/**/*.js', '!public/custom/**/*.spec.js'], dest: 'public/js'}, + {expand: true, flatten: true, src: ['public/custom/**/*.css'], dest: 'public/css'}, + ], + }, + // Unit test files + unit: { + files: [ + {expand: true, flatten: true, src: ['node_modules/angular-mocks/angular-mocks.js'], dest: 'public/test_frameworks'}, + {expand: true, flatten: true, src: ['node_modules/jasmine-jquery/lib/jasmine-jquery.js'], dest: 'public/test_frameworks'}, + ], + }, + }, + + // Unit test configuration + karma: { + unit: { + configFile: 'karma.conf.js', + }, + }, + + // Combine multiple files to make it easier to import and use them + concat: { + options: { + sourceMap: true, + }, + css: { + files: { + 'public/css/ui.css': ['mapped/ui/*.css'], + }, + }, + vinyldns: { + options: { + banner: license, + separator: ';\n', + process: (src, filepath) => { + return '/* Source: ' + filepath + '*/\n' + + src.replace(/(^|\n)[ \t]*('use strict'|"use strict");?\s*/g, '$1').replace(/\/\*.+?\*\//gs, '').replace(/\n{2,}/g, '\n'); + }, + }, + files: { + 'public/js/vinyldns.js': [ + 'public/lib/**/*.module.js', + 'public/lib/**/*.js', + '!public/lib/**/*.spec.js', + ], + }, + }, + ui_javascript: { + options: { + separator: ';\n', + process: (src, filepath) => { + return '/* Source: ' + filepath + ' */\n' + + src.replace(/(^|\n)[ \t]*('use strict'|"use strict");?\s*/g, '$1'); + }, + }, + files: { + 'public/js/ui.js': ['mapped/ui/**/*.js'], + }, + }, + ui_css: { + options: { + separator: '\n', + process: (src, filepath) => { + return '/* Source: ' + filepath + ' */\n' + src; + }, + }, + files: { + 'public/css/ui.css': ['mapped/ui/**/*.css'], + }, + }, + }, + + clean: { + js: ['public/js/*', '!public/js/custom.js'], + css: ['public/css/*', '!public/css/{theme-overrides,vinyldns}.css'], + mapped: ['mapped/'], + fonts: ['public/fonts'], + unit: ['public/test_frameworks/*'], + }, + }); + + // Default task + grunt.registerTask('default', ['clean', 'copy:main', 'concat', 'clean:mapped']); + // Unit tests + grunt.registerTask('unit', ['default', 'copy:unit', 'karma:unit', 'clean:unit']); }; diff --git a/modules/portal/README.md b/modules/portal/README.md index faa7fb02a..b1ccca194 100644 --- a/modules/portal/README.md +++ b/modules/portal/README.md @@ -1,19 +1,24 @@ # VinylDNS Portal -Supplies a UI for and offers authentication into Vinyl, a DNSaaS offering. + +Supplies a UI for and offers authentication into VinylDNS. # Running Unit Tests + First, startup sbt: `sbt`. Next, you can run all tests by simply running `test`, or you can run an individual test by running `test-only *MySpec` # Running Frontend Tests + The front end tests can be run from the `test/portal/fuctional` directory by simply running `make`. # Building Locally 1. You must have npm, if you don't have npm, follow instructions here . -2. Run `npm install` to install all dependencies, this includes those needed for testing. If you just want to run the portal then `npm install --production` would suffice -3. You must have grunt, if you don't have grunt, run `npm install -g grunt`. Then run `grunt default` from the root of the portal project +2. Run `npm install` to install all dependencies, this includes those needed for testing. If you just want to run the + portal then `npm install --production` would suffice +3. You must have grunt, if you don't have grunt, run `npm install -g grunt`. Then run `grunt default` from the root of + the portal project 4. Create a local.conf file in the portal conf folder for your settings if desired. 5. Follow the instructions for building vinyl locally on the vinyl readme 6. Start vinyl with `sbt run`. Vinyl will start on localhost on port 9000. @@ -21,20 +26,25 @@ The front end tests can be run from the `test/portal/fuctional` directory by sim 8. In a web browser go to localhost:9001 # Working locally -Often times as a developer you want to work with the portal locally in a "real" setting against your own LDAP -server. If your LDAP server requires SSL certs, you will need to create a trust store (or register the -SSL certs on your local machine). If you create a trust store, you will need to make the trust store -available so that you can start the portal locally and test. + +Often times as a developer you want to work with the portal locally in a "real" setting against your own LDAP server. If +your LDAP server requires SSL certs, you will need to create a trust store (or register the SSL certs on your local +machine). If you create a trust store, you will need to make the trust store available so that you can start the portal +locally and test. 1. Create a trust store and save your certs. -1. Pass the trust store in when you start sbt. This can be on the command line like... -`sbt -Djavax.net.ssl.trustStore="./private/trustStore.jks"` +1. Pass the trust store in when you start sbt. This can be on the command line like... + `sbt -Djavax.net.ssl.trustStore="./private/trustStore.jks"` # Updating the trustStore Certificates -When some event causes the LDAP lookup to fail because of SSL certificate issues, follow -the following steps to update the trustStore with the new certificates. -- Get the new certificate with `openssl s_client -connect :`. This will display the certificate on the screen. -- Copy everything from `-----BEGIN CERTIFICATE-----` to `-----END CERTIFICATE-----` including the begin and end markers to the clipboard. + +When some event causes the LDAP lookup to fail because of SSL certificate issues, follow the following steps to update +the `trustStore` with the new certificates. + +- Get the new certificate with `openssl s_client -connect :`. This will display the certificate on + the screen. +- Copy everything from `-----BEGIN CERTIFICATE-----` to `-----END CERTIFICATE-----` including the begin and end markers + to the clipboard. - Open a new file in your favorite text editor - Paste the clipboard contents into the file - Save the file and give it a name to be used in the next steps. (ex. `new-ad-ssl-cert.pem`) diff --git a/modules/portal/app/views/groups/groupDetail.scala.html b/modules/portal/app/views/groups/groupDetail.scala.html index 1e62fae2c..e7c3862d8 100644 --- a/modules/portal/app/views/groups/groupDetail.scala.html +++ b/modules/portal/app/views/groups/groupDetail.scala.html @@ -109,9 +109,7 @@ } @plugins = { - - - + } @main(rootAccountName)("MembershipController")("Group")(content)(plugins) diff --git a/modules/portal/app/views/groups/groups.scala.html b/modules/portal/app/views/groups/groups.scala.html index 0815d216d..76d57f5f2 100644 --- a/modules/portal/app/views/groups/groups.scala.html +++ b/modules/portal/app/views/groups/groups.scala.html @@ -266,9 +266,7 @@ } @plugins = { - - - + } @main(rootAccountName)("GroupsController")("Groups")(content)(plugins) diff --git a/modules/portal/app/views/login.scala.html b/modules/portal/app/views/login.scala.html index b1d0e5168..9b402ec6e 100644 --- a/modules/portal/app/views/login.scala.html +++ b/modules/portal/app/views/login.scala.html @@ -13,8 +13,9 @@ - - + + + diff --git a/modules/portal/app/views/main.scala.html b/modules/portal/app/views/main.scala.html index 32542466f..64c088544 100644 --- a/modules/portal/app/views/main.scala.html +++ b/modules/portal/app/views/main.scala.html @@ -4,7 +4,7 @@ - + @pageHeader | VinylDNS @@ -12,16 +12,14 @@ - + - - - - - + + + + - @@ -97,8 +95,7 @@ @header(rootAccountName)(request) @pageContent -

VinylDNS (version @{meta.version})

- +

VinylDNS (version @{meta.version})

@@ -142,65 +139,22 @@
Are you sure you want to regenerate your credentials?  -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + @pagePlugins diff --git a/modules/portal/app/views/recordsets/recordSets.scala.html b/modules/portal/app/views/recordsets/recordSets.scala.html index a822b7573..cc55d54c7 100644 --- a/modules/portal/app/views/recordsets/recordSets.scala.html +++ b/modules/portal/app/views/recordsets/recordSets.scala.html @@ -374,9 +374,7 @@ } @plugins = { - - - + } @main(rootAccountName)("RecordSetsController")("RecordSets")(content)(plugins) diff --git a/modules/portal/app/views/setOidcSession.scala.html b/modules/portal/app/views/setOidcSession.scala.html index a22304352..76bfda744 100644 --- a/modules/portal/app/views/setOidcSession.scala.html +++ b/modules/portal/app/views/setOidcSession.scala.html @@ -11,7 +11,8 @@ + Finishing login, if not redirected, click this link - +@* *@ diff --git a/modules/portal/app/views/systemMessage.scala.html b/modules/portal/app/views/systemMessage.scala.html index 8a87b34e0..687fcddc3 100644 --- a/modules/portal/app/views/systemMessage.scala.html +++ b/modules/portal/app/views/systemMessage.scala.html @@ -12,8 +12,9 @@ - - + + + diff --git a/modules/portal/app/views/zones/zoneDetail.scala.html b/modules/portal/app/views/zones/zoneDetail.scala.html index de9a53dcc..dc8b06c33 100644 --- a/modules/portal/app/views/zones/zoneDetail.scala.html +++ b/modules/portal/app/views/zones/zoneDetail.scala.html @@ -68,10 +68,7 @@ } @plugins = { - - - - + } @main(rootAccountName)("RecordsController")("Zone")(content)(plugins) diff --git a/modules/portal/app/views/zones/zones.scala.html b/modules/portal/app/views/zones/zones.scala.html index ed0cee221..52ab8b759 100644 --- a/modules/portal/app/views/zones/zones.scala.html +++ b/modules/portal/app/views/zones/zones.scala.html @@ -267,9 +267,7 @@ } @plugins = { - - - + } @main(rootAccountName)("ZonesController")("Zones")(content)(plugins) diff --git a/modules/portal/karma.conf.js b/modules/portal/karma.conf.js index 66f8af984..ac09db9d3 100644 --- a/modules/portal/karma.conf.js +++ b/modules/portal/karma.conf.js @@ -14,18 +14,7 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ - 'javascripts/moment.min.js', - 'gentelella/vendors/jquery/dist/jquery.min.js', - 'gentelella/vendors/bootstrap/dist/js/bootstrap.min.js', - 'gentelella/vendors/bootstrap-daterangepicker/daterangepicker.js', - 'javascripts/angular.min.js', - 'test_frameworks/*.js', - 'lib/services/**/*.js', - 'lib/controllers/**/*.js', - 'lib/directives/**/*.js', - 'lib/batch-change/batch-change.module.js', - 'lib/*.js', - 'lib/batch-change/*.js', + '*.js', //fixtures {pattern: 'mocks/*.json', watched: true, served: true, included: false} ], diff --git a/modules/portal/package.json b/modules/portal/package.json index a6cc2fedc..048c047d0 100644 --- a/modules/portal/package.json +++ b/modules/portal/package.json @@ -1,25 +1,19 @@ { "name": "vinyl-portal", "version": "1.0.0", - "description": "Self-service DNS offering", "main": "index.js", "directories": { "test": "test" }, "dependencies": { - "@uirouter/angularjs": "0.3.2", - "angular": ">=1.6.0", - "angular-animate": "1.6.1", - "angular-ui-bootstrap": "2.5.6", - "bootstrap": "^4.3.1", - "font-awesome": "4.7.0", - "gentelella": "https://github.com/colorlibhq/gentelella.git#eea0150c65d51cb1f1a03f18bcc3bfa2a8db9312", + "angular": "^1.8.2", + "bootstrap": "^3.4.1", + "font-awesome": "^4.7.0", + "gentelella": "git+https://github.com/colorlibhq/gentelella.git#eea0150c65d51cb1f1a03f18bcc3bfa2a8db9312", "grunt": "^1.0.4", - "grunt-angular-templates": "^1.1.0", "grunt-contrib-clean": "^1.0.0", "grunt-contrib-copy": "^1.0.0", - "grunt-copy": "^0.1.0", - "grunt-injector": "^1.1.0", + "grunt-contrib-concat": "^2.0.0", "moment": "^2.24.0", "puppeteer": "^1.11.0" }, @@ -29,7 +23,7 @@ "grunt-mocha-phantomjs": "^3.0.0", "jasmine-core": "^2.99.1", "jasmine-jquery": "2.1.1", - "jquery": "^3.4.1", + "jquery": "^2.2.4", "karma": "^2.0.5", "karma-chrome-launcher": "^2.2.0", "karma-jasmine": "^1.0.2", diff --git a/modules/portal/prepare-portal.sh b/modules/portal/prepare-portal.sh index 83fc6ee9f..316dfcddd 100755 --- a/modules/portal/prepare-portal.sh +++ b/modules/portal/prepare-portal.sh @@ -1,12 +1,10 @@ #!/usr/bin/env bash -DIR=$( cd $(dirname $0) ; pwd -P ) +set -euo pipefail +DIR=$( cd "$(dirname "$0")" ; pwd -P ) -cd $DIR - -npm install -f +cd "${DIR}" +npm install -f --no-audit --no-fund npm install grunt -g -f -grunt default -$DIR/../../utils/add-license-headers.sh -d=$DIR/public/lib -f=js -cd - +grunt default diff --git a/modules/portal/public/css/theme-overrides.css b/modules/portal/public/css/theme-overrides.css index 136ac2214..b1742a8d9 100644 --- a/modules/portal/public/css/theme-overrides.css +++ b/modules/portal/public/css/theme-overrides.css @@ -1,3 +1,18 @@ +/* +* Copyright 2018 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the \"License\"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an \"AS IS\" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ body { color: #000; } diff --git a/modules/portal/public/css/vinyldns.css b/modules/portal/public/css/vinyldns.css index e0100d93f..88768b9a2 100644 --- a/modules/portal/public/css/vinyldns.css +++ b/modules/portal/public/css/vinyldns.css @@ -1,3 +1,18 @@ +/* +* Copyright 2018 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the \"License\"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an \"AS IS\" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ [ng\:cloak], [ng-cloak], [data-ng-cloak], [x-ng-cloak], .ng-cloak, .x-ng-cloak { display: none !important; } diff --git a/modules/portal/public/js/custom.js b/modules/portal/public/js/custom.js index f560e432e..cec2a53c0 100644 --- a/modules/portal/public/js/custom.js +++ b/modules/portal/public/js/custom.js @@ -1,35 +1,39 @@ -var CURRENT_URL = window.location.href.split('#')[0].split('?')[0], - $MENU_INDENT = $('#menu_indent'), +/* +* Copyright 2018 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the \"License\"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an \"AS IS\" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +const $MENU_INDENT = $('#menu_indent'), $MENU_DEDENT = $('#menu_dedent'), $MENU_MOBILE = $('#menu_mobile'), $BODY = $('body'), $SIDEBAR_MENU = $('#sidebar-menu'); $MENU_INDENT.on('click', function() { - console.log('clicked - menu indent'); - console.log("HEEEEEEY!") + $SIDEBAR_MENU.find('li.active-sm ul').show(); + $SIDEBAR_MENU.find('li.active-sm').addClass('active').removeClass('active-sm'); - $SIDEBAR_MENU.find('li.active-sm ul').show(); - $SIDEBAR_MENU.find('li.active-sm').addClass('active').removeClass('active-sm'); - - - $BODY.addClass('nav-md').removeClass('nav-sm'); + $BODY.addClass('nav-md').removeClass('nav-sm'); }); $MENU_DEDENT.on('click', function() { - console.log('clicked - menu dedent'); - console.log("HEEEEEEY!") + $SIDEBAR_MENU.find('li.active ul').hide(); + $SIDEBAR_MENU.find('li.active').addClass('active-sm').removeClass('active'); - $SIDEBAR_MENU.find('li.active ul').hide(); - $SIDEBAR_MENU.find('li.active').addClass('active-sm').removeClass('active'); - - $BODY.addClass('nav-sm').removeClass('nav-md'); + $BODY.addClass('nav-sm').removeClass('nav-md'); }); $MENU_MOBILE.on('click', function() { - console.log('clicked - menu indent'); - console.log("HEEEEEEY!") - if ($BODY.hasClass('nav-md')) { $SIDEBAR_MENU.find('li.active ul').hide(); $SIDEBAR_MENU.find('li.active').addClass('active-sm').removeClass('active'); diff --git a/modules/portal/public/lib/controllers/controller.zones.js b/modules/portal/public/lib/controllers/controller.zones.js index 6813cdfb6..8159be547 100644 --- a/modules/portal/public/lib/controllers/controller.zones.js +++ b/modules/portal/public/lib/controllers/controller.zones.js @@ -67,7 +67,7 @@ angular.module('controller.zones', []) }); $scope.canAccessGroup = function(groupId) { - return $scope.myGroupIds.indexOf(groupId) > -1; + return $scope.myGroupIds !== "undefined" && $scope.myGroupIds.indexOf(groupId) > -1; }; $scope.canAccessZone = function(accessLevel) { diff --git a/modules/portal/public/lib/dns-change/dns-change-new.controller.js b/modules/portal/public/lib/dns-change/dns-change-new.controller.js index a0234c0ba..2f6993d19 100644 --- a/modules/portal/public/lib/dns-change/dns-change-new.controller.js +++ b/modules/portal/public/lib/dns-change/dns-change-new.controller.js @@ -56,7 +56,6 @@ }; $scope.confirmSubmit = function(form) { - console.log(form.$error) if(form.$invalid){ form.$setSubmitted(); $scope.formStatus = "pendingSubmit"; @@ -93,7 +92,6 @@ if(entry.changeType == 'DeleteRecordSet' && entry.record) { var recordDataEmpty = true; for (var attr in entry.record) { - console.log(entry.record[attr]) if (entry.record[attr] != undefined && entry.record[attr].toString().length > 0) { recordDataEmpty = false } diff --git a/modules/portal/public/lib/dns-change/dns-change.spec.js b/modules/portal/public/lib/dns-change/dns-change.spec.js index c626d8e7d..b6dfab9d9 100644 --- a/modules/portal/public/lib/dns-change/dns-change.spec.js +++ b/modules/portal/public/lib/dns-change/dns-change.spec.js @@ -549,7 +549,7 @@ describe('BatchChange', function(){ expect(this.scope.change.cname).toBeUndefined(); expect(form.cname.$valid).toBe(false); }); - } + }); describe('Directive: IPv4 validation', function(){ var form; diff --git a/modules/portal/public/lib/recordset/recordsets.controller.js b/modules/portal/public/lib/recordset/recordsets.controller.js index 431f77fa8..b23ccb9a4 100644 --- a/modules/portal/public/lib/recordset/recordsets.controller.js +++ b/modules/portal/public/lib/recordset/recordsets.controller.js @@ -37,7 +37,6 @@ function success(response) { recordsPaging.next = response.data.nextId; updateRecordDisplay(response.data['recordSets']); - console.log($scope.records); } return recordsService @@ -57,7 +56,6 @@ }); function handleError(error, type) { - console.log(error); var alert = utilityService.failure(error, type); $scope.alerts.push(alert); $scope.processing = false; diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2a4d1eb18..756cce23d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -16,7 +16,7 @@ object Dependencies { lazy val awsV = "1.11.423" lazy val jaxbV = "2.3.0" lazy val ip4sV = "1.1.1" - lazy val fs2V = "2.4.4" + lazy val fs2V = "2.4.5" lazy val ficusV = "1.4.3" lazy val apiDependencies = Seq( @@ -71,7 +71,7 @@ object Dependencies { "com.sun.xml.bind" % "jaxb-impl" % jaxbV, "ch.qos.logback" % "logback-classic" % "1.0.7", "io.dropwizard.metrics" % "metrics-jvm" % "3.2.2", - "co.fs2" %% "fs2-core" % "2.3.0", + "co.fs2" %% "fs2-core" % fs2V, "javax.xml.bind" % "jaxb-api" % "2.3.0", "javax.activation" % "activation" % "1.1.1", ) diff --git a/project/Resolvers.scala b/project/Resolvers.scala deleted file mode 100644 index 76f79021f..000000000 --- a/project/Resolvers.scala +++ /dev/null @@ -1,10 +0,0 @@ -import sbt._ - -object Resolvers { - - lazy val additionalResolvers = Seq( - "dnvriend at bintray" at "https://dl.bintray.com/dnvriend/maven", - "bintray" at "https://jcenter.bintray.com", - "DynamoDBLocal" at "https://s3-us-west-2.amazonaws.com/dynamodb-local/release" - ) -} diff --git a/project/assembly.sbt b/project/assembly.sbt deleted file mode 100644 index d6540b84c..000000000 --- a/project/assembly.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.8") diff --git a/project/plugins.sbt b/project/plugins.sbt index a7b25462c..139dab396 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,6 +2,10 @@ logLevel := Level.Warn resolvers += "Flyway" at "https://flywaydb.org/repo" +addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.18") + +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0") + addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.0") addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") diff --git a/project/protoc.sbt b/project/protoc.sbt deleted file mode 100644 index 29a56e7bd..000000000 --- a/project/protoc.sbt +++ /dev/null @@ -1 +0,0 @@ -addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.18") diff --git a/quickstart/api/Dockerfile b/quickstart/api/Dockerfile deleted file mode 100644 index 5bd5c9696..000000000 --- a/quickstart/api/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# Build VinylDNS API if the JAR doesn't already exist -ARG VINYLDNS_VERSION=latest -FROM vinyldns/build:base-build as base-build -ARG DOCKER_FILE_PATH -COPY . /build/ -WORKDIR /build - -## Run the build if we don't already have a vinyldns.jar -RUN mkdir -p /opt/vinyldns/conf && \ - if [ -f assembly/vinyldns.jar ]; then cp assembly/vinyldns.jar /opt/vinyldns/; fi && \ - if [ ! -f /opt/vinyldns/vinyldns.jar ]; then \ - env SBT_OPTS="-XX:+UseConcMarkSweepGC -Xmx4G -Xms1G" \ - sbt -Dbuild.scalafmtOnCompile=false -Dbuild.lintOnCompile=fase ";project api;coverageOff;assembly" \ - && cp assembly/vinyldns.jar /opt/vinyldns/; \ - fi - -FROM adoptopenjdk/openjdk11:jdk-11.0.8_10-alpine - -RUN apk add --update --no-cache bash - -COPY --from=base-build /opt/vinyldns /opt/vinyldns - -# This will set the vinyldns version, make sure to have this in config... version = ${?VINYLDNS_VERSION} -ARG VINYLDNS_VERSION="0.0.0-local-dev" -ENV VINYLDNS_VERSION=$VINYLDNS_VERSION - -RUN mkdir -p /opt/vinyldns/lib_extra - -# Mount the volume for config file and lib extras -# Note: These volume names are used in the build.sbt -VOLUME ["/opt/vinyldns/lib_extra/", "/opt/vinyldns/conf"] - -EXPOSE 9000 - -ENTRYPOINT ["java", "-Dconfig.file=/opt/vinyldns/conf/vinyldns.conf", \ - "-Dlogback.configurationFile=/opt/vinyldns/conf/logback.xml",\ - "-Dvinyldns.base-version=${VINYLDNS_VERSION}",\ - "-cp", "/opt/vinyldns/lib_extra/*",\ - "-jar", "/opt/vinyldns/vinyldns.jar"] diff --git a/quickstart/api/Makefile b/quickstart/api/Makefile deleted file mode 100644 index 35f2a6705..000000000 --- a/quickstart/api/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -SHELL=bash -IMAGE_NAME=vinyldns/api:local-dev -ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) - -# Check that the required version of make is being used -REQ_MAKE_VER:=3.82 -ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) - $(error The version of MAKE $(REQ_MAKE_VER) or higher is required; you are running $(MAKE_VERSION)) -endif - -# Extract arguments for `make run` -EXTRACT_ARGS=true -ifeq (run,$(firstword $(MAKECMDGOALS))) - EXTRACT_ARGS=true -endif -ifeq ($(EXTRACT_ARGS),true) - # use the rest as arguments for "run" - WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) -endif -ifdef $(WITH_ARGS) - ARG_SEPARATOR=-- -endif - -%: - @: - -.ONESHELL: - -.PHONY: all build run - -all: build run - -build: - @set -euo pipefail - cd ../.. - docker build -t $(IMAGE_NAME) --build-arg DOCKER_FILE_PATH="$$(realpath --relative-to="." "$(ROOT_DIR)")" -f "$(ROOT_DIR)/Dockerfile" . - -run: - @set -euo pipefail - docker network create --driver bridge vinyldns_net &> /dev/null || true - test -t 1 && USE_TTY="-t" - docker run -i $${USE_TTY} --rm $(DOCKER_PARAMS) --network vinyldns_net -v "$$(pwd)/application.conf:/opt/vinyldns/conf/vinyldns.conf" -v "$$(pwd)/logback.xml:/opt/vinyldns/conf/logback.xml" -p 9000:9000 $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) - -run-bg: - @set -euo pipefail - docker stop $(IMAGE_NAME) &> /dev/null || true - docker rm $(IMAGE_NAME) &> /dev/null || true - docker network create --driver bridge vinyldns_net &> /dev/null || true - test -t 1 && USE_TTY="-t" - docker run -d $${USE_TTY}--name $(IMAGE_NAME) --rm $(DOCKER_PARAMS) --network vinyldns_net -v "$$(pwd)/application.conf:/opt/vinyldns/conf/vinyldns.conf" -v "$$(pwd)/logback.xml:/opt/vinyldns/conf/logback.xml" -p 9001:9001 $(IMAGE_NAME) -- /bin/bash diff --git a/quickstart/api/application.conf b/quickstart/api/application.conf deleted file mode 100644 index 58903a17e..000000000 --- a/quickstart/api/application.conf +++ /dev/null @@ -1,346 +0,0 @@ -vinyldns { - - base-version = "0.0.0-local-dev" - version = ${vinyldns.base-version} # default to the base version if not overridden - version = ${?VINYLDNS_VERSION} # override the base version via env var - - # How often to any particular zone can be synchronized in milliseconds - sync-delay = 10000 - sync-delay = ${?SYNC_DELAY} - - # If we should start up polling for change requests, set this to false for the inactive cluster - processing-disabled = false - processing-disabled = ${?PROCESSING_DISABLED} - - # Number of records that can be in a zone - max-zone-size = 60000 - max-zone-size = ${?MAX_ZONE_SIZE} - - # Types of unowned records that users can access in shared zones - shared-approved-types = ["A", "AAAA", "CNAME", "PTR", "TXT"] - - # Batch change settings - batch-change-limit = 1000 - batch-change-limit = ${?BATCH_CHANGE_LIMIT} - manual-batch-review-enabled = true - manual-batch-review-enabled = ${?MANUAL_BATCH_REVIEW_ENABLED} - scheduled-changes-enabled = true - scheduled-changes-enabled = ${?SCHEDULED_CHANGES_ENABLED} - multi-record-batch-change-enabled = true - multi-record-batch-change-enabled = ${?MULTI_RECORD_BATCH_CHANGE_ENABLED} - - # configured backend providers - backend { - # Use "default" when dns backend legacy = true - # otherwise, use the id of one of the connections in any of your backends - default-backend-id = "default" - - # this is where we can save additional backends - backend-providers = [ - { - class-name = "vinyldns.api.backend.dns.DnsBackendProviderLoader" - settings = { - legacy = false - backends = [ - { - id = "default" - zone-connection = { - name = "vinyldns." - key-name = "vinyldns." - key-name = ${?DEFAULT_DNS_KEY_NAME} - key = "nzisn+4G2ldMn0q1CV3vsg==" - key = ${?DEFAULT_DNS_KEY_SECRET} - primary-server = "127.0.0.1" - primary-server = ${?DEFAULT_DNS_ADDRESS} - } - transfer-connection = { - name = "vinyldns." - key-name = "vinyldns." - key-name = ${?DEFAULT_DNS_KEY_NAME} - key = "nzisn+4G2ldMn0q1CV3vsg==" - key = ${?DEFAULT_DNS_KEY_SECRET} - primary-server = "127.0.0.1" - primary-server = ${?DEFAULT_DNS_ADDRESS} - }, - tsig-usage = "always" - }, - { - id = "func-test-backend" - zone-connection = { - name = "vinyldns." - key-name = "vinyldns." - key-name = ${?DEFAULT_DNS_KEY_NAME} - key = "nzisn+4G2ldMn0q1CV3vsg==" - key = ${?DEFAULT_DNS_KEY_SECRET} - primary-server = "127.0.0.1" - primary-server = ${?DEFAULT_DNS_ADDRESS} - } - transfer-connection = { - name = "vinyldns." - key-name = "vinyldns." - key-name = ${?DEFAULT_DNS_KEY_NAME} - key = "nzisn+4G2ldMn0q1CV3vsg==" - key = ${?DEFAULT_DNS_KEY_SECRET} - primary-server = "127.0.0.1" - primary-server = ${?DEFAULT_DNS_ADDRESS} - }, - tsig-usage = "always" - } - ] - } - } - ] - } - - - queue { - class-name = "vinyldns.sqs.queue.SqsMessageQueueProvider" - - messages-per-poll = 10 - polling-interval = 250.millis - - settings { - # AWS access key and secret. - access-key = "test" - access-key = ${?AWS_ACCESS_KEY} - secret-key = "test" - secret-key = ${?AWS_SECRET_ACCESS_KEY} - - # Regional endpoint to make your requests (eg. 'us-west-2', 'us-east-1', etc.). This is the region where your queue is housed. - signing-region = "us-east-1" - signing-region = ${?SQS_REGION} - - # Endpoint to access queue - service-endpoint = "http://vinyldns-integration:19003/" - service-endpoint = ${?SQS_SERVICE_ENDPOINT} - - # Queue name. Should be used in conjunction with service endpoint, rather than using a queue url which is subject to change. - queue-name = "vinyldns" - queue-name = ${?SQS_QUEUE_NAME} - } - } - - email { - class-name = "vinyldns.api.notifier.email.EmailNotifierProvider" - class-name = ${?EMAIL_CLASS_NAME} - settings = { - from = "VinylDNS " - } - } - - sns { - class-name = "vinyldns.apadi.notifier.sns.SnsNotifierProvider" - class-name = ${?SNS_CLASS_NAME} - settings { - topic-arn = "arn:aws:sns:us-east-1:000000000000:batchChanges" - topic-arn = ${?SNS_TOPIC_ARN} - access-key = "test" - access-key = ${?SNS_ACCESS_KEY} - secret-key = "test" - secret-key = ${?SNS_SECRET_KEY} - service-endpoint = "http://vinyldns-integration:19003" - service-endpoint = ${?SNS_SERVICE_ENDPOINT} - signing-region = "us-east-1" - signing-region = ${?SNS_REGION} - } - } - - rest { - host = "0.0.0.0" - port = 9000 - port=${?API_SERVICE_PORT} - } - - - approved-name-servers = [ - "172.17.42.1.", - "ns1.parent.com." - "ns1.parent.com1." - "ns1.parent.com2." - "ns1.parent.com3." - "ns1.parent.com4." - ] - - crypto { - type = "vinyldns.core.crypto.NoOpCrypto" - } - - data-stores = ["mysql"] - - mysql { - settings { - # JDBC Settings, these are all values in scalikejdbc-config, not our own - # these must be overridden to use MYSQL for production use - # assumes a docker or mysql instance running locally - name = "vinyldns" - name = ${?DATABASE_NAME} - driver = "org.h2.Driver" - driver = ${?JDBC_DRIVER} - migration-url = "jdbc:h2:mem:vinyldns;MODE=MYSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE;IGNORECASE=TRUE;INIT=RUNSCRIPT FROM 'classpath:test/ddl.sql'" - migration-url = ${?JDBC_MIGRATION_URL} - url = "jdbc:h2:mem:vinyldns;MODE=MYSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=TRUE;IGNORECASE=TRUE;INIT=RUNSCRIPT FROM 'classpath:test/ddl.sql'" - url = ${?JDBC_URL} - user = "sa" - user = ${?JDBC_USER} - password = "" - password = ${?JDBC_PASSWORD} - # see https://github.com/brettwooldridge/HikariCP - connection-timeout-millis = 1000 - idle-timeout = 10000 - max-lifetime = 600000 - maximum-pool-size = 20 - minimum-idle = 20 - register-mbeans = true - } - # Repositories that use this data store are listed here - repositories { - zone { - # no additional settings for now - } - batch-change { - # no additional settings for now - } - user { - - } - record-set { - - } - group { - - } - membership { - - } - group-change { - - } - zone-change { - - } - record-change { - - } - } - } - - backends = [] - - - # FQDNs / IPs that cannot be modified via VinylDNS - # regex-list used for all record types except PTR - # ip-list used exclusively for PTR records - high-value-domains = { - regex-list = [ - "high-value-domain.*" # for testing - ] - ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing - "192.0.2.252", - "192.0.2.253", - "fd69:27cc:fe91:0:0:0:0:ffff", - "fd69:27cc:fe91:0:0:0:ffff:0" - ] - } - - # FQDNs / IPs / zone names that require manual review upon submission in batch change interface - # domain-list used for all record types except PTR - # ip-list used exclusively for PTR records - manual-review-domains = { - domain-list = [ - "needs-review.*" - ] - ip-list = [ - "192.0.1.254", - "192.0.1.255", - "192.0.2.254", - "192.0.2.255", - "192.0.3.254", - "192.0.3.255", - "192.0.4.254", - "192.0.4.255", - "fd69:27cc:fe91:0:0:0:ffff:1", - "fd69:27cc:fe91:0:0:0:ffff:2", - "fd69:27cc:fe92:0:0:0:ffff:1", - "fd69:27cc:fe92:0:0:0:ffff:2", - "fd69:27cc:fe93:0:0:0:ffff:1", - "fd69:27cc:fe93:0:0:0:ffff:2", - "fd69:27cc:fe94:0:0:0:ffff:1", - "fd69:27cc:fe94:0:0:0:ffff:2" - ] - zone-name-list = [ - "zone.requires.review." - "zone.requires.review1." - "zone.requires.review2." - "zone.requires.review3." - "zone.requires.review4." - ] - } - - # FQDNs / IPs that cannot be modified via VinylDNS - # regex-list used for all record types except PTR - # ip-list used exclusively for PTR records - high-value-domains = { - regex-list = [ - "high-value-domain.*" # for testing - ] - ip-list = [ - # using reverse zones in the vinyldns/bind9 docker image for testing - "192.0.1.252", - "192.0.1.253", - "192.0.2.252", - "192.0.2.253", - "192.0.3.252", - "192.0.3.253", - "192.0.4.252", - "192.0.4.253", - "fd69:27cc:fe91:0:0:0:0:ffff", - "fd69:27cc:fe91:0:0:0:ffff:0", - "fd69:27cc:fe92:0:0:0:0:ffff", - "fd69:27cc:fe92:0:0:0:ffff:0", - "fd69:27cc:fe93:0:0:0:0:ffff", - "fd69:27cc:fe93:0:0:0:ffff:0", - "fd69:27cc:fe94:0:0:0:0:ffff", - "fd69:27cc:fe94:0:0:0:ffff:0" - ] - } - - global-acl-rules = [ - { - group-ids: ["global-acl-group-id"], - fqdn-regex-list: [".*shared[0-9]{1}."] - }, - { - group-ids: ["another-global-acl-group"], - fqdn-regex-list: [".*ok[0-9]{1}."] - } - ] -} - -akka { - loglevel = "INFO" - loggers = ["akka.event.slf4j.Slf4jLogger"] - logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" - logger-startup-timeout = 30s - - actor { - provider = "akka.actor.LocalActorRefProvider" - } -} - -akka.http { - server { - # The time period within which the TCP binding process must be completed. - # Set to `infinite` to disable. - bind-timeout = 5s - - # Show verbose error messages back to the client - verbose-error-messages = on - } - - parsing { - # Spray doesn't like the AWS4 headers - illegal-header-warnings = on - } -} - diff --git a/quickstart/api/logback.xml b/quickstart/api/logback.xml deleted file mode 100644 index a7f4b877d..000000000 --- a/quickstart/api/logback.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - %d [test] %-5p | \(%logger{4}:%line\) | %msg %n - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/quickstart/docker-compose.yml b/quickstart/docker-compose.yml index f0bdf785d..5460696dc 100644 --- a/quickstart/docker-compose.yml +++ b/quickstart/docker-compose.yml @@ -15,7 +15,7 @@ services: context: ../ dockerfile: test/api/integration/Dockerfile args: - VINYLDNS_VERSION: "${VINYLDNS_VERSION}" + VINYLDNS_BASE_VERSION: "${VINYLDNS_BASE_VERSION}" environment: RUN_SERVICES: "deps-only tail-logs" LOCALSTACK_EXT_HOSTNAME: *integration_hostname @@ -30,12 +30,12 @@ services: image: "vinyldns/api:${VINYLDNS_IMAGE_VERSION}" build: context: ../ - dockerfile: quickstart/api/Dockerfile + dockerfile: build/docker/api/Dockerfile args: - VINYLDNS_VERSION: "${VINYLDNS_VERSION}" - DOCKER_FILE_PATH: "quickstart/api" + VINYLDNS_VERSION: "${VINYLDNS_IMAGE_VERSION}" + DOCKER_FILE_PATH: "../build/docker/api" volumes: - - ./api/application.conf:/opt/vinyldns/conf/vinyldns.conf + - ../build/docker/api/application.conf:/opt/vinyldns/conf/vinyldns.conf env_file: .env ports: @@ -48,14 +48,16 @@ services: image: "vinyldns/portal:${VINYLDNS_IMAGE_VERSION}" build: context: ../ - dockerfile: quickstart/portal/Dockerfile + dockerfile: build/docker/portal/Dockerfile + args: + DOCKER_FILE_PATH: "../build/docker/portal" + VINYLDNS_VERSION: "${VINYLDNS_IMAGE_VERSION}" env_file: .env ports: - "${PORTAL_PORT}:${PORTAL_PORT}" volumes: - - ./portal/application.ini:/opt/vinyldns/conf/application.ini - - ./portal/application.conf:/opt/vinyldns/conf/application.conf + - ../build/docker/portal/application.conf:/opt/vinyldns/conf/application.conf depends_on: - api - ldap diff --git a/quickstart/portal/Dockerfile b/quickstart/portal/Dockerfile deleted file mode 100644 index 0727fe393..000000000 --- a/quickstart/portal/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM vinyldns/build:base-build-portal as builder -ARG VINYLDNS_VERSION="0.0.0-local-dev" - -COPY . /vinyldns - -WORKDIR /vinyldns -RUN cp /build/node_modules.tar.xz /vinyldns/modules/portal && \ - cd /vinyldns/modules/portal && tar Jxf node_modules.tar.xz && \ - cd /vinyldns - -RUN sbt "set version in ThisBuild := \"${VINYLDNS_VERSION}\"; project portal; preparePortal" -RUN sbt "set version in ThisBuild := \"${VINYLDNS_VERSION}\"; project portal; universal:packageZipTarball" - -FROM adoptopenjdk/openjdk11:jdk-11.0.8_10-alpine - -RUN apk add --update --no-cache bash - -COPY --from=builder /vinyldns/modules/portal/target/universal/portal.tgz / - -RUN mkdir -p /opt && \ - tar -xzf /portal.tgz && \ - mv /portal /opt/vinyldns && \ - mkdir -p /opt/vinyldns/lib_extra - -# This will set the vinyldns version, make sure to have this in config... version = ${?VINYLDNS_VERSION} -ENV VINYLDNS_VERSION=$VINYLDNS_VERSION - -# Mount the volume for config file and lib extras -# Note: These volume names are used in the build.sbt -VOLUME ["/opt/vinyldns/lib_extra/", "/opt/vinyldns/conf"] - -EXPOSE 9001 - -ENTRYPOINT ["/opt/vinyldns/bin/portal"] diff --git a/quickstart/portal/application.conf b/quickstart/portal/application.conf deleted file mode 100644 index 40ae2c83a..000000000 --- a/quickstart/portal/application.conf +++ /dev/null @@ -1,62 +0,0 @@ -LDAP { - # For OpenLDAP, this would be a full DN to the admin for LDAP / user that can see all users - user = "cn=admin,dc=planetexpress,dc=com" - - # Password for the admin account - password = "GoodNewsEveryone" - - # Keep this as an empty string for OpenLDAP - domain = "" - - # This will be the name of the LDAP field that carries the user's login id (what they enter in the username in login form) - userNameAttribute = "uid" - - # For organization, leave empty for this demo, the domainName is what matters, and that is the LDAP structure - # to search for users that require login - searchBase = [ - {organization = "", domainName = "ou=people,dc=planetexpress,dc=com"}, - ] - context { - initialContextFactory = "com.sun.jndi.ldap.LdapCtxFactory" - securityAuthentication = "simple" - - # Note: The following assumes a purely docker setup, using container_name = vinyldns-ldap - providerUrl = "ldap://vinyldns-ldap:19004" - providerUrl = ${?LDAP_PROVIDER_URL} - } - - # This is only needed if keeping vinyldns user store in sync with ldap (to auto lock out users who left your - # company for example) - user-sync { - enabled = false - hours-polling-interval = 1 - } -} - -# Note: This MUST match the API or strange errors will ensure, NoCrypto should not be used for production -crypto { - type = "vinyldns.core.crypto.NoOpCrypto" -} - -http.port = 9001 -http.port = ${?PORTAL_PORT} - -data-stores = ["mysql"] - -# Note: The default mysql settings assume a local docker compose setup with mysql named vinyldns-mysql -# follow the configuration guide to point to your mysql -# Only 3 repositories are needed for portal: user, task, user-change -mysql { - repositories { - user { - } - task { - } - user-change { - } - } -} - -# You generate this yourself following https://www.playframework.com/documentation/2.7.x/ApplicationSecret -play.http.secret.key = "rpkTGtoJvLIdIV?WU=0@yW^x:pcEGyAt`^p/P3G0fpbj9:uDnD@caSjCDqA0@tB=" -play.http.secret.key = ${?PLAY_HTTP_SECRET_KEY} diff --git a/quickstart/portal/application.ini b/quickstart/portal/application.ini deleted file mode 100644 index ddbb62aa0..000000000 --- a/quickstart/portal/application.ini +++ /dev/null @@ -1,3 +0,0 @@ -# uncomment to set custom trustStore -# don't forget to mount trustStore to docker image -#-Djavax.net.ssl.trustStore=/opt/docker/conf/trustStore.jks diff --git a/quickstart/quickstart-vinyldns.sh b/quickstart/quickstart-vinyldns.sh index 2b53c0ae6..04bd826e8 100755 --- a/quickstart/quickstart-vinyldns.sh +++ b/quickstart/quickstart-vinyldns.sh @@ -51,6 +51,7 @@ function wait_for_url() { fi done } + function is_running() { if (docker ps --format "{{.Image}}" | grep -q "$1"); then return 0 @@ -77,8 +78,14 @@ set -a source "${DIR}/.env" set +a -# Set defaults and parse args +# The version of VinylDNS docker image to run export VINYLDNS_VERSION=latest +# The base/starting version of VinylDNS docker build image to use (vinyldns/build:) +export VINYLDNS_BASE_VERSION=latest +# The version of the images to build +export VINYLDNS_IMAGE_VERSION=${VINYLDNS_VERSION} + +# Defaults TIMEOUT=60 DOCKER_COMPOSE_CONFIG="${DIR}/docker-compose.yml" SERVICE="" @@ -124,6 +131,8 @@ while [[ $# -gt 0 ]]; do ;; -v | --version-tag) export VINYLDNS_VERSION=$2 + export VINYLDNS_BASE_VERSION=${VINYLDNS_VERSION} + export VINYLDNS_IMAGE_VERSION=${VINYLDNS_VERSION} shift shift ;; @@ -142,18 +151,22 @@ if [[ $RESET_DOCKER -eq 1 ]] || [[ $CLEAN -eq 1 ]]; then fi fi -export VINYLDNS_IMAGE_VERSION=${VINYLDNS_VERSION} -if [ -n "${BUILD}" ] || [ -n "$(docker images vinyldns/portal:local-dev --format '{{.Repository}}:{{.Tag}}')" ]; then - VINYLDNS_IMAGE_VERSION="local-dev" -fi - # Update images if requested if [[ $UPDATE -eq 1 ]]; then - echo "${F_YELLOW}Removing local docker images tagged ${F_RESET}'${VINYLDNS_IMAGE_VERSION}'${F_YELLOW}...${F_RESET}" + echo "${F_YELLOW}Removing any local docker containers tagged ${F_RESET}'${VINYLDNS_IMAGE_VERSION}'${F_YELLOW}...${F_RESET}" "${DIR}/../utils/clean-vinyldns-containers.sh" - docker rmi "vinyldns/build:base-test-integration-${VINYLDNS_IMAGE_VERSION}" - docker rmi "vinyldns/portal:${VINYLDNS_IMAGE_VERSION}" - docker rmi "vinyldns/api:${VINYLDNS_IMAGE_VERSION}" + + echo "${F_YELLOW}Removing any local docker images tagged ${F_RESET}'${VINYLDNS_IMAGE_VERSION}'${F_YELLOW}...${F_RESET}" + docker rmi "vinyldns/build:base-test-integration-${VINYLDNS_IMAGE_VERSION}" &> /dev/null || true + docker rmi "vinyldns/portal:${VINYLDNS_IMAGE_VERSION}" &> /dev/null || true + docker rmi "vinyldns/api:${VINYLDNS_IMAGE_VERSION}" &> /dev/null || true + echo "${F_GREEN}Removed all local docker images and containers tagged ${F_RESET}'${VINYLDNS_IMAGE_VERSION}'${F_YELLOW}...${F_RESET}" +fi + + +if [ -n "${BUILD}" ] || [ -n "$(docker images vinyldns/portal:local-dev --format '{{.Repository}}:{{.Tag}}')" ]; then + VINYLDNS_IMAGE_VERSION="local-dev" + export VINYLDNS_VERSION=${VINYLDNS_IMAGE_VERSION} fi if [ -n "${BUILD}" ]; then @@ -162,14 +175,16 @@ else echo "Starting VinylDNS (${VINYLDNS_IMAGE_VERSION}) the background..." fi docker-compose -f "${DOCKER_COMPOSE_CONFIG}" up ${BUILD} -d ${SERVICE} || ( - echo -e "${F_RED}Sorry, there was an error starting VinylDNS :-(\nTry resetting any existing containers with:\n\t${F_RESET}'$0 --reset'" - exit 1 + echo -e "${F_RED}Sorry, there was an error starting VinylDNS :-(\nTry resetting any existing containers with:\n\t${F_RESET}'$0 --reset'"; \ + exit 1; \ ) -echo -wait_for_api -wait_for_portal -echo +if is_running "vinyldns/portal" || is_running "vinyldns/api"; then + echo + wait_for_api + wait_for_portal + echo +fi if is_running "vinyldns/portal"; then echo "${F_GREEN}VinylDNS started! You can connect to the portal via ${F_RESET}${VINYLDNS_PORTAL_URL}" diff --git a/test/api/functional/Makefile b/test/api/functional/Makefile index c8b6b575c..7704fcee6 100644 --- a/test/api/functional/Makefile +++ b/test/api/functional/Makefile @@ -11,20 +11,21 @@ ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) endif # Extract arguments for `make run` -EXTRACT_ARGS=true +EXTRACT_ARGS=false ifeq (run,$(firstword $(MAKECMDGOALS))) - EXTRACT_ARGS=true + EXTRACT_ARGS=true endif ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" - WITH_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) - # ...and turn them into do-nothing targets - $(eval $(WITH_ARGS):;@:) + WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif -ifdef WITH_ARGS +ifneq ($(WITH_ARGS),) ARG_SEPARATOR=-- endif +%: + @: + .ONESHELL: .PHONY: all build run run-local @@ -38,10 +39,10 @@ build: run: @set -euo pipefail - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -i $${USE_TTY} --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) run-local: @set -euo pipefail - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -i $${USE_TTY} --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp -v "$(ROOT_DIR)/../../../modules/api/src/test/functional:/functional_test" $(IMAGE_NAME) -- $(WITH_ARGS) diff --git a/test/api/integration/Dockerfile b/test/api/integration/Dockerfile index ffa44f6cc..ae0241bfd 100644 --- a/test/api/integration/Dockerfile +++ b/test/api/integration/Dockerfile @@ -1,6 +1,7 @@ +ARG VINYLDNS_BASE_VERSION=latest + # Build VinylDNS API if the JAR doesn't already exist -ARG VINYLDNS_VERSION=latest -FROM vinyldns/build:base-build-${VINYLDNS_VERSION} as base-build +FROM vinyldns/build:base-build as base-build ARG DOCKERFILE_PATH="test/api/integration" COPY "${DOCKERFILE_PATH}/vinyldns.*" /opt/vinyldns/ COPY . /build/ @@ -14,8 +15,8 @@ RUN if [ -f assembly/vinyldns.jar ]; then cp assembly/vinyldns.jar /opt/vinyldns && cp assembly/vinyldns.jar /opt/vinyldns/; \ fi -# Build the testing image, copying data from `vinyldns-api` -FROM vinyldns/build:base-test-integration-${VINYLDNS_VERSION} +# Build the testing image, copying data from `base-build` +FROM vinyldns/build:base-test-integration-${VINYLDNS_BASE_VERSION} SHELL ["/bin/bash","-c"] ARG DOCKERFILE_PATH COPY --from=base-build /opt/vinyldns /opt/vinyldns diff --git a/test/api/integration/Makefile b/test/api/integration/Makefile index 7b00c9f3a..0ee3b5d3b 100644 --- a/test/api/integration/Makefile +++ b/test/api/integration/Makefile @@ -10,7 +10,7 @@ ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) endif # Extract arguments for `make run` -EXTRACT_ARGS=true +EXTRACT_ARGS=false ifeq (run,$(firstword $(MAKECMDGOALS))) EXTRACT_ARGS=true endif @@ -18,8 +18,8 @@ ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif -ifdef WITH_ARGS - ARG_SEPARATOR=-- +ifneq ($(WITH_ARGS),) + ARG_SEPARATOR=-- endif %: @@ -38,13 +38,13 @@ build: run: @set -euo pipefail - test -t 1 && USE_TTY="-t" - docker run -i $${USE_TTY} --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) + USE_TTY="" && test -t 1 && USE_TTY="-t" + docker run -i $${USE_TTY} --rm $(DOCKER_PARAMS) $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) run-bg: @set -euo pipefail docker stop $(IMAGE_NAME) &> /dev/null || true - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -d $${USE_TTY} --name $(IMAGE_NAME) --rm $(DOCKER_PARAMS) -e RUN_SERVICES="deps-only tail-logs" -p 19001-19003:19001-19003 -p 19001:19001/udp $(IMAGE_NAME) stop-bg: @@ -53,5 +53,5 @@ stop-bg: run-local: @set -euo pipefail - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -i $${USE_TTY} --rm $(DOCKER_PARAMS) -p 9000:9000 -p 19001-19003:19001-19003 -p 19001:19001/udp -v "$(ROOT_DIR)/../../..:/build" $(IMAGE_NAME) -- $(WITH_ARGS) diff --git a/test/portal/functional/Makefile b/test/portal/functional/Makefile index 97ff9d870..140456856 100644 --- a/test/portal/functional/Makefile +++ b/test/portal/functional/Makefile @@ -10,20 +10,21 @@ ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) endif # Extract arguments for `make run` -EXTRACT_ARGS=true +EXTRACT_ARGS=false ifeq (run,$(firstword $(MAKECMDGOALS))) EXTRACT_ARGS=true endif ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" - WITH_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) - # ...and turn them into do-nothing targets - $(eval $(WITH_ARGS):;@:) + WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif -ifdef WITH_ARGS - ARG_SEPARATOR=-- +ifneq ($(WITH_ARGS),) + ARG_SEPARATOR=-- endif +%: + @: + .ONESHELL: .PHONY: all build run run-local @@ -37,10 +38,10 @@ build: run: @set -euo pipefail - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -i $${USE_TTY} --rm $(IMAGE_NAME) -- $(WITH_ARGS) run-local: @set -euo pipefail - test -t 1 && USE_TTY="-t" + USE_TTY="" && test -t 1 && USE_TTY="-t" docker run -i $${USE_TTY} --rm -v "$$(pwd)/../../../modules/portal:/functional_test" $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) diff --git a/utils/admin/Dockerfile b/utils/admin/Dockerfile index e36eb6e55..1701fac80 100644 --- a/utils/admin/Dockerfile +++ b/utils/admin/Dockerfile @@ -1,18 +1,23 @@ FROM znly/protoc:0.4.0 as pbcompile +WORKDIR /vinyldns # Needs to protoc compile modules/core/src/main/protobuf/VinylDNSProto.proto -COPY VinylDNSProto.proto /vinyldns/target/ +COPY modules/core/src/main/protobuf/VinylDNSProto.proto /vinyldns # Create a compiled protobuf in /vinyldns/target -RUN protoc --version && \ - protoc --proto_path=/vinyldns/target --python_out=/vinyldns/target /vinyldns/target/VinylDNSProto.proto +RUN mkdir -p /vinyldns/python && \ + protoc --proto_path=/vinyldns --python_out=/vinyldns/python /vinyldns/VinylDNSProto.proto FROM python:3.7-alpine - -RUN pip install mysql-connector-python - -COPY --from=pbcompile /vinyldns/target /app/ -COPY update-support-user.py /app/update-support-user.py - +ARG DOCKERFILE_PATH WORKDIR /app +RUN pip install mysql-connector-python==8.0.27 + +ENV DB_USER="root" DB_PASS="pass" DB_HOST="vinyldns-integration" DB_NAME="vinyldns" DB_PORT="19002" + +COPY --from=pbcompile /vinyldns/python . +COPY ${DOCKERFILE_PATH}/update-support-user.py . +RUN chmod 755 update-support-user.py + +ENTRYPOINT ["./update-support-user.py"] diff --git a/quickstart/portal/Makefile b/utils/admin/Makefile similarity index 52% rename from quickstart/portal/Makefile rename to utils/admin/Makefile index 95e1ddf5f..2f9e422fa 100644 --- a/quickstart/portal/Makefile +++ b/utils/admin/Makefile @@ -1,7 +1,7 @@ SHELL=bash -IMAGE_TAG=$(shell awk -F'"' '{print $$2}' ../../../version.sbt) -IMAGE_NAME=vinyldns/portal +IMAGE_NAME=vinyldns-update-user ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +RELATIVE_ROOT_DIR:=$(shell realpath --relative-to=../.. $(ROOT_DIR)) # Check that the required version of make is being used REQ_MAKE_VER:=3.82 @@ -10,7 +10,7 @@ ifneq ($(REQ_MAKE_VER),$(firstword $(sort $(MAKE_VERSION) $(REQ_MAKE_VER)))) endif # Extract arguments for `make run` -EXTRACT_ARGS=true +EXTRACT_ARGS=false ifeq (run,$(firstword $(MAKECMDGOALS))) EXTRACT_ARGS=true endif @@ -18,8 +18,8 @@ ifeq ($(EXTRACT_ARGS),true) # use the rest as arguments for "run" WITH_ARGS ?= $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) endif -ifdef $(WITH_ARGS) - ARG_SEPARATOR=-- +ifneq ($(WITH_ARGS),) + ARG_SEPARATOR=-- endif %: @@ -34,18 +34,10 @@ all: build run build: @set -euo pipefail cd ../.. - docker build -t $(IMAGE_NAME) -f "$(ROOT_DIR)/Dockerfile" . + docker build -t $(IMAGE_NAME) --build-arg DOCKERFILE_PATH="$(RELATIVE_ROOT_DIR)" -f "$(ROOT_DIR)/Dockerfile" . run: @set -euo pipefail docker network create --driver bridge vinyldns_net &> /dev/null || true - test -t 1 && USE_TTY="-t" - docker run -i $${USE_TTY} --rm $(DOCKER_PARAMS) --network vinyldns_net -p 9001:9001 $(IMAGE_NAME) $(ARG_SEPARATOR) $(WITH_ARGS) - -run-bg: - @set -euo pipefail - docker stop $(IMAGE_NAME) &> /dev/null || true - docker rm $(IMAGE_NAME) &> /dev/null || true - docker network create --driver bridge vinyldns_net &> /dev/null || true - test -t 1 && USE_TTY="-t" - docker run -d $${USE_TTY} --name $(IMAGE_NAME) --rm $(DOCKER_PARAMS) --network vinyldns_net -p 9001:9001 $(IMAGE_NAME) -- /bin/bash + USE_TTY="" && test -t 1 && USE_TTY="-t" + docker run -i $${USE_TTY} --network vinyldns_net --rm $(DOCKER_PARAMS) $(IMAGE_NAME) $(WITH_ARGS) diff --git a/utils/admin/update-support-user.py b/utils/admin/update-support-user.py index 4157f054b..70d58a342 100644 --- a/utils/admin/update-support-user.py +++ b/utils/admin/update-support-user.py @@ -5,8 +5,12 @@ import sys import os # arguments +if len(sys.argv) != 3: + print("USAGE: update-support-user.py ") + exit(1) + user_name = sys.argv[1] -make_support = sys.argv[2][0].upper()=='T' if len(sys.argv) >= 2 else None +make_support = sys.argv[2][0].upper() == 'T' # environment variables to connect to database db_user = os.environ.get('DB_USER') diff --git a/utils/update-support-user.sh b/utils/update-support-user.sh index ad8e841b3..c08269a9e 100755 --- a/utils/update-support-user.sh +++ b/utils/update-support-user.sh @@ -90,14 +90,6 @@ then fi # Copy the proto definition to the Docker context and build -cp "${VINYL_ROOT}/modules/core/src/main/protobuf/VinylDNSProto.proto" "${WORK_DIR}/admin" -docker build -t vinyldns/admin "${WORK_DIR}/admin" -rm "${WORK_DIR}/admin/VinylDNSProto.proto" - -docker run -it --rm \ - -e "DB_USER=$DB_USER" \ - -e "DB_PASS=$DB_PASS" \ - -e "DB_HOST=$DB_HOST" \ - -e "DB_NAME=$DB_NAME" \ - -e "DB_PORT=$DB_PORT" \ - vinyldns/admin:latest /app/update-support-user.py "$VINYL_USER" "$MAKE_SUPPORT" +cd admin +make build +make run DOCKER_PARAMS="-e \"DB_USER=$DB_USER\" -e \"DB_PASS=$DB_PASS\" -e \"DB_HOST=$DB_HOST\" -e \"DB_NAME=$DB_NAME\" -e \"DB_PORT=$DB_PORT\"" WITH_ARGS="\"$VINYL_USER\" \"$MAKE_SUPPORT\""