Build optimizations (#883)

A few specific build optimizations:

1. Consolidated `dockerComposeUp` to only use a single `root/docker/docker-compose.yml` instead of each module having its own docker-compose files.  This eliminates additional waits for docker containers to startup and stop, as well as reduces memory consumption during the build
2. Cleaned up `VinylDNSSpec` - I noticed that this spec was taking 3 minutes to run!  I discovered that the way we were mocking the `WSClient` was largely to blame.  Was able to get tests to run in **16 SECONDS** using a library called `mock-ws`.  This is where we see most savings.
3. Added back `dynamodb-local` instead of running it in `localstack`.  Integration tests for dynamodb were very slow in localstack.  This added an additional 20-30 second improvement.

After doing several tests locally running the following command...

```
> SECONDS=0; sbt verify; echo "DURATION = $SECONDS SECONDS"
```

Current master took 535 seconds to run; with these optimizations it took **211 SECONDS** - that is a 60% improvement.

The initial Travis builds reported a run time of 13 minutes as opposed to 19 minutes; this would save some 6 minutes off of Travis build times (or 30% improvement).

build.sbt

@@ -136,7 +136,6 @@ lazy val apiDockerSettings = Seq(
     ExecCmd("RUN", "apk", "add", "--update", "--no-cache", "netcat-openbsd", "bash"),
     Cmd("USER", "1001:0") // switch back to the daemon user
   ),
-  composeFile := baseDirectory.value.getAbsolutePath + "/../../docker/docker-compose.yml"
 )
 
 lazy val portalDockerSettings = Seq(
@@ -219,7 +218,7 @@ lazy val allApiSettings = Revolver.settings ++ Defaults.itSettings ++
   scalaStyleSettings
 
 lazy val api = (project in file("modules/api"))
-  .enablePlugins(JavaAppPackaging, DockerComposePlugin, AutomateHeaderPlugin)
+  .enablePlugins(JavaAppPackaging, AutomateHeaderPlugin)
   .configs(IntegrationTest)
   .settings(allApiSettings)
   .settings(headerSettings(IntegrationTest))
@@ -230,7 +229,7 @@ lazy val api = (project in file("modules/api"))
   .dependsOn(sqs % "compile->compile;it->it")
 
 val killDocker = TaskKey[Unit]("killDocker", "Kills all vinyldns docker containers")
-lazy val root = (project in file(".")).enablePlugins(AutomateHeaderPlugin)
+lazy val root = (project in file(".")).enablePlugins(DockerComposePlugin, AutomateHeaderPlugin)
   .configs(IntegrationTest)
   .settings(headerSettings(IntegrationTest))
   .settings(sharedSettings)
@@ -290,7 +289,7 @@ lazy val core = (project in file("modules/core")).enablePlugins(AutomateHeaderPl
   )
 
 lazy val dynamodb = (project in file("modules/dynamodb"))
-  .enablePlugins(DockerComposePlugin, AutomateHeaderPlugin)
+  .enablePlugins(AutomateHeaderPlugin)
   .configs(IntegrationTest)
   .settings(sharedSettings)
   .settings(headerSettings(IntegrationTest))
@@ -308,7 +307,7 @@ lazy val dynamodb = (project in file("modules/dynamodb"))
   .settings(name := "dynamodb")
 
 lazy val mysql = (project in file("modules/mysql"))
-  .enablePlugins(DockerComposePlugin, AutomateHeaderPlugin)
+  .enablePlugins(AutomateHeaderPlugin)
   .configs(IntegrationTest)
   .settings(sharedSettings)
   .settings(headerSettings(IntegrationTest))
@@ -324,7 +323,7 @@ lazy val mysql = (project in file("modules/mysql"))
   .settings(name := "mysql")
 
 lazy val sqs = (project in file("modules/sqs"))
-  .enablePlugins(DockerComposePlugin, AutomateHeaderPlugin)
+  .enablePlugins(AutomateHeaderPlugin)
   .configs(IntegrationTest)
   .settings(sharedSettings)
   .settings(headerSettings(IntegrationTest))
@@ -520,12 +519,9 @@ addCommandAlias("validate", "; root/clean; " +
   "root/compile;root/test:compile;root/it:compile"
 )
 
-addCommandAlias("verify", "; project root; killDocker; " +
-  "project api; dockerComposeUp; project dynamodb; dockerComposeUp; project mysql; dockerComposeUp; " +
-  "project sqs; dockerComposeUp;" +
+addCommandAlias("verify", "; project root; killDocker; dockerComposeUp; " +
   "project root; coverage; " +
-  "all core/test dynamodb/test mysql/test api/test dynamodb/it:test mysql/it:test api/it:test portal/test " +
-  "sqs/test sqs/it:test; " +
+  "all test it:test; " +
   "project root; coverageReport; coverageAggregate; killDocker"
 )
 

docker/.env

@@ -11,7 +11,7 @@ PORTAL_PORT=9001
 PLAY_HTTP_SECRET_KEY=change-this-for-prod
 VINYLDNS_BACKEND_URL=http://vinyldns-api:9000
 SQS_ENDPOINT=http://vinyldns-localstack:19007
-DYNAMODB_ENDPOINT=http://vinyldns-localstack:19000
+DYNAMODB_ENDPOINT=http://vinyldns-dynamodb:8000
 MYSQL_ENDPOINT=vinyldns-mysql:3306
 USER_TABLE_NAME=users
 USER_CHANGE_TABLE_NAME=userChange

docker/api/docker.conf

@@ -232,6 +232,8 @@ vinyldns {
 
   scheduled-changes-enabled = true
 
+  multi-record-batch-change-enabled = true
+
   global-acl-rules = [
     {
       group-ids: ["global-acl-group-id"],

docker/docker-compose-func-test.yml

@@ -24,16 +24,27 @@ services:
     logging:
       driver: none
 
+  dynamodb-local:
+    image: cnadiminti/dynamodb-local:2017-02-16
+    container_name: "vinyldns-dynamodb"
+    env_file:
+      .env
+    logging:
+      driver: none
+    ports:
+      - "19000:8000"
+    command: "--sharedDb --inMemory"
+
   localstack:
     image: localstack/localstack:0.10.4
     container_name: "vinyldns-localstack"
     ports:
-      - "19000:19000"
       - "19006:19006"
       - "19007:19007"
     environment:
-      - SERVICES=dynamodb:19000,sns:19006,sqs:19007
+      - SERVICES=sns:19006,sqs:19007
       - START_WEB=0
+      - HOSTNAME_EXTERNAL=vinyldns-localstack
 
   # this file is copied into the target directory to get the jar!  won't run in place as is!
   api:

docker/docker-compose.yml

@@ -1,14 +1,14 @@
 version: "3.0"
 services:
   mysql:
-    image: "mysql:5.7"
+    image: mysql:5.7<skipPull>
     env_file:
       .env
     ports:
       - "19002:3306"
 
   bind9:
-    image: "vinyldns/bind9:0.0.4"
+    image: vinyldns/bind9:0.0.4<skipPull>
     env_file:
       .env
     ports:
@@ -18,24 +18,31 @@ services:
       - ./bind9/etc:/var/cache/bind/config
       - ./bind9/zones:/var/cache/bind/zones
 
+  dynamodb-local:
+    image: cnadiminti/dynamodb-local:2017-02-16<skipPull>
+    env_file:
+      .env
+    ports:
+      - "19000:8000"
+    command: "--sharedDb --inMemory"
+
   localstack:
     image: localstack/localstack:0.10.4<skipPull>
     ports:
-      - "19000:19000"
       - "19006:19006"
       - "19007:19007"
     environment:
-      - SERVICES=dynamodb:19000,sns:19006,sqs:19007
+      - SERVICES=sns:19006,sqs:19007
       - START_WEB=0
 
   mail:
-    image: flaviovs/mock-smtp
+    image: flaviovs/mock-smtp:0.0.2<skipPull>
     ports:
       - "19025:25"
     volumes:
       - ./email:/var/lib/mock-smtp
 
   ldap:
-    image: rroemhild/test-openldap
+    image: rroemhild/test-openldap:latest<skipPull>
     ports:
       - "19008:389"

docker/functest/run.sh

@@ -47,25 +47,32 @@ cd /app
 find . -name "*.pyc" -delete
 find . -name "__pycache__" -delete
 
+result=0
 # If PROD_ENV is not true, we are in a local docker environment so do not skip anything
 if [ "${PROD_ENV}" = "true" ]; then
     # -m plays havoc with -k, using variables is a headache, so doing this by hand
     # run parallel tests first (not serial)
     echo "./run-tests.py live_tests -n2 -v -m \"not skip_production and not serial and not multi_record_enabled\" -v --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=False"
     ./run-tests.py live_tests -n2 -v -m "not skip_production and not serial and not multi_record_enabled" --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=False
-    if [ $? -eq 0 ]; then
+    result=$?
+    if [ $result -eq 0 ]; then
       # run serial tests second (serial marker)
       echo "./run-tests.py live_tests -n0 -v -m \"not skip_production and serial and not multi_record_enabled\" -v --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=True"
       ./run-tests.py live_tests -n0 -v -m "not skip_production and serial and not multi_record_enabled" --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=True
+      result=$?
     fi
 else
     # run parallel tests first (not serial)
     echo "./run-tests.py live_tests -n2 -v -m \"not serial and not multi_record_disabled\" --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=False"
     ./run-tests.py live_tests -n2 -v -m "not serial and not multi_record_disabled" --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=False
-
-    if [ $? -eq 0 ]; then
+    result=$?
+    if [ $result -eq 0 ]; then
       # run serial tests second (serial marker)
       echo "./run-tests.py live_tests -n0 -v -m \"serial and not multi_record_disabled\" --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=True"
       ./run-tests.py live_tests -n0 -v -m "serial and not multi_record_disabled" --url=${VINYLDNS_URL} --dns-ip=${DNS_IP} ${TEST_PATTERN} --teardown=True
+      result=$?
     fi
 fi
+
+exit $result
+

modules/dynamodb/docker/docker-compose.yml

@@ -1,9 +0,0 @@
-version: "3.0"
-services:
-  localstack:
-    image: localstack/localstack:0.10.4<skipPull>
-    ports:
-      - "19003:19003"
-    environment:
-      - SERVICES=dynamodb:19003
-      - START_WEB=0

modules/dynamodb/src/it/resources/application.conf

@@ -6,35 +6,35 @@ dynamodb {
   settings {
     key = "vinyldnsTest"
     secret = "notNeededForDynamoDbLocal"
-    endpoint = "http://127.0.0.1:19003"
+    endpoint = "http://127.0.0.1:19000"
     region = "us-east-1"
   }
 
   repositories {
     record-change {
       table-name = "recordchange-startup-test"
-      provisioned-reads = 40
-      provisioned-writes = 30
+      provisioned-reads = 100
+      provisioned-writes = 100
     }
     zone-change {
       table-name = "zonechange-startup-test"
-      provisioned-reads = 40
-      provisioned-writes = 30
+      provisioned-reads = 100
+      provisioned-writes = 100
     }
     group {
       table-name = "groups-startup-test"
-      provisioned-reads = 40
-      provisioned-writes = 30
+      provisioned-reads = 100
+      provisioned-writes = 100
     }
     group-change {
       table-name = "groupchanges-startup-test"
-      provisioned-reads = 40
-      provisioned-writes = 30
+      provisioned-reads = 100
+      provisioned-writes = 100
     }
     membership {
       table-name = "memberships-startup-test"
-      provisioned-reads = 40
-      provisioned-writes = 30
+      provisioned-reads = 100
+      provisioned-writes = 100
     }
   }
 }

modules/dynamodb/src/it/scala/vinyldns/dynamodb/repository/DynamoDBIntegrationSpec.scala

@@ -28,7 +28,7 @@ trait DynamoDBIntegrationSpec
     with Inspectors {
 
   // port is defined in the docker/docker-compose.yml file for dynamodb
-  val dynamoIntegrationConfig: DynamoDBDataStoreSettings = getDynamoConfig(19003)
+  val dynamoIntegrationConfig: DynamoDBDataStoreSettings = getDynamoConfig(19000)
   val logger: Logger = LoggerFactory.getLogger("DynamoDBIntegrationSpec")
 
   // only used for teardown

modules/mysql/docker/conf/config-file.cnf

@@ -1,3 +0,0 @@
-[mysqld]
-# Turn this on to record queries
-general_log = 1

modules/mysql/docker/docker-compose.yml

@@ -1,11 +0,0 @@
-version: "3.0"
-services:
-  mysql-module:
-    image: "mysql:5.7"
-    environment:
-    - MYSQL_ROOT_PASSWORD=pass # do not use quotes around the environment variables!!!
-    - MYSQL_ROOT_HOST=% # this is required as mysql is currently locked down to localhost
-    ports:
-    - "19004:3306"
-    volumes:
-    - ./conf:/etc/mysql/conf.d

modules/mysql/src/it/resources/application.conf

@@ -5,10 +5,10 @@ mysql {
     # JDBC Settings, these are all values in scalikejdbc-config, not our own
     # these must be overridden to use MYSQL for production use
     # assumes a docker or mysql instance running locally
-    name = "vinyldns"
+    name = "vinyldns2"
     driver = "org.mariadb.jdbc.Driver"
-    migration-url = "jdbc:mariadb://localhost:19004/"
-    url = "jdbc:mariadb://localhost:19004/vinyldns"
+    migration-url = "jdbc:mariadb://localhost:19002/"
+    url = "jdbc:mariadb://localhost:19002/vinyldns2"
     user = "root"
     password = "pass"
 
@@ -40,10 +40,10 @@ queue {
   messages-per-poll = 10
 
   settings = {
-    name = "vinyldns"
+    name = "vinyldns2"
     driver = "org.mariadb.jdbc.Driver"
-    migration-url = "jdbc:mariadb://localhost:19004/?user=root&password=pass"
-    url = "jdbc:mariadb://localhost:19004/vinyldns?user=root&password=pass"
+    migration-url = "jdbc:mariadb://localhost:19002/?user=root&password=pass"
+    url = "jdbc:mariadb://localhost:19002/vinyldns2?user=root&password=pass"
     user = "root"
     password = "pass"
 

modules/sqs/docker/custom.conf

@@ -1,21 +0,0 @@
-node-address {
-  protocol = http
-  host = "localhost"
-  port = 19005
-  context-path = ""
-}
-
-rest-sqs {
-  enabled = true
-  bind-port = 9324
-  bind-hostname = "0.0.0.0"
-  // Possible values: relaxed, strict
-  sqs-limits = relaxed
-}
-
-queues {
-  sqs {
-    defaultVisibilityTimeout = 10 seconds
-    receiveMessageWait = 0 seconds
-  }
-}

modules/sqs/docker/docker-compose.yml

@@ -1,9 +0,0 @@
-version: "3.0"
-services:
-  localstack:
-    image: localstack/localstack:0.10.4<skipPull>
-    ports:
-      - "19005:19005"
-    environment:
-      - SERVICES=sqs:19005
-      - START_WEB=0

modules/sqs/src/it/resources/application.conf

@@ -6,7 +6,7 @@ sqs {
     access-key = "x"
     secret-key = "x"
     signing-region = "x"
-    service-endpoint = "http://localhost:19005/"
+    service-endpoint = "http://localhost:19007/"
     queue-name = "sqs-override-name"
   }
 }

modules/sqs/src/it/scala/vinyldns/sqs/queue/SqsMessageQueueProviderIntegrationSpec.scala

@@ -34,7 +34,7 @@ class SqsMessageQueueProviderIntegrationSpec extends WordSpec with Matchers {
           |    settings {
           |      access-key = "x"
           |      signing-region = "x"
-          |      service-endpoint = "http://localhost:19005/"
+          |      service-endpoint = "http://localhost:19007/"
           |      queue-name = "queue-name"
           |    }
           |    """.stripMargin)
@@ -56,7 +56,7 @@ class SqsMessageQueueProviderIntegrationSpec extends WordSpec with Matchers {
           |      access-key = "x"
           |      secret-key = "x"
           |      signing-region = "x"
-          |      service-endpoint = "http://localhost:19005/"
+          |      service-endpoint = "http://localhost:19007/"
           |      queue-name = "new-queue"
           |    }
           |    """.stripMargin)
@@ -81,7 +81,7 @@ class SqsMessageQueueProviderIntegrationSpec extends WordSpec with Matchers {
           |      access-key = "x"
           |      secret-key = "x"
           |      signing-region = "x"
-          |      service-endpoint = "http://localhost:19005/"
+          |      service-endpoint = "http://localhost:19007/"
           |      queue-name = "bad*queue*name"
           |    }
           |    """.stripMargin)
@@ -101,7 +101,7 @@ class SqsMessageQueueProviderIntegrationSpec extends WordSpec with Matchers {
           |      access-key = "x"
           |      secret-key = "x"
           |      signing-region = "x"
-          |      service-endpoint = "http://localhost:19005/"
+          |      service-endpoint = "http://localhost:19007/"
           |      queue-name = "queue.fifo"
           |    }
           |    """.stripMargin)
@@ -123,7 +123,7 @@ class SqsMessageQueueProviderIntegrationSpec extends WordSpec with Matchers {
           |      access-key = "x"
           |      secret-key = "x"
           |      signing-region = "x"
-          |      service-endpoint = "http://localhost:19005/"
+          |      service-endpoint = "http://localhost:19007/"
           |      queue-name = "new-queue"
           |    }
           |    """.stripMargin)

project/Dependencies.scala

@@ -117,6 +117,7 @@ object Dependencies {
     "io.netty"                  %  "netty-transport-native-unix-common" % "4.1.37.Final" % "test",
     "com.nimbusds"              % "oauth2-oidc-sdk"                 % "6.5",
     "com.nimbusds"              % "nimbus-jose-jwt"                 % "7.0",
-    "co.fs2"                    %% "fs2-core"                       % fs2V
+    "co.fs2"                    %% "fs2-core"                       % fs2V,
+    "de.leanovate.play-mockws"  %% "play-mockws" % "2.7.1"          % "test"
   )
 }