Boot load repos in api (#214)

* conf changes to dynamically load

* have boot dynamically load repos based on config

* update sbt version

.travis.yml

@@ -18,6 +18,7 @@ cache:
   directories:
     - $HOME/.ivy2/cache
     - $HOME/.sbt
+  timeout: 900
 
 install:
   - rvm use 2.2.8 --install --fuzzy
@@ -26,6 +27,8 @@ install:
 
 before_cache:
   # Cleanup the cached directories to avoid unnecessary cache updates (https://www.scala-sbt.org/1.0/docs/Travis-CI-with-sbt.html)
+  - sudo chown -R travis:travis $HOME/.sbt
+  - sudo chown -R travis:travis $HOME/.ivy2/cache
   - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
   - find $HOME/.sbt        -name "*.lock"               -print -delete
 

docker/api/docker.conf

@@ -43,7 +43,7 @@ vinyldns {
     type = "vinyldns.core.crypto.NoOpCrypto"
   }
 
-  data-stores = ["mysql"]
+  data-stores = ["mysql", "dynamodb"]
 
   mysql {
     settings {
@@ -75,69 +75,59 @@ vinyldns {
     }
   }
 
-  # default settings point to the docker compose setup
-  dynamo {
-    key = "x"
-    key = ${?AWS_ACCESS_KEY}
-    secret = "x"
-    secret = ${?AWS_SECRET_ACCESS_KEY}
-    endpoint = "http://vinyldns-dynamodb:8000"
-    endpoint = ${?DYNAMODB_ENDPOINT}
-  }
-
-  zoneChanges {
-    dynamo {
-      tableName = "zoneChange"
-      provisionedReads = 30
-      provisionedWrites = 30
+  dynamodb {
+    settings {
+      # default settings point to the docker compose setup
+      key = "x"
+      key = ${?AWS_ACCESS_KEY}
+      secret = "x"
+      secret = ${?AWS_SECRET_ACCESS_KEY}
+      endpoint = "http://vinyldns-dynamodb:8000"
+      endpoint = ${?DYNAMODB_ENDPOINT}
     }
-  }
 
-  recordSet {
-    dynamo {
-      tableName = "recordSet"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
+    repositories {
+      zone-change {
+        table-name = "zoneChange"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
 
-  recordChange {
-    dynamo {
-      tableName = "recordChange"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
+      record-set {
+        table-name = "recordSet"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
 
-  users {
-    dynamo {
-      tableName = "users"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
+      record-change {
+        table-name = "recordChange"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
 
-  groups {
-    dynamo {
-      tableName = "groups"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
+      user {
+        table-name = "users"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
 
-  groupChanges {
-    dynamo {
-      tableName = "groupChanges"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
+      group {
+        table-name = "groups"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
 
-  membership {
-    dynamo {
-      tableName = "membership"
-      provisionedReads = 30
-      provisionedWrites = 30
+      group-change {
+        table-name = "groupChanges"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
+
+      membership {
+        table-name = "membership"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
     }
   }
 

modules/api/src/it/resources/application.conf

@@ -31,24 +31,42 @@ vinyldns {
     }
   }
 
-  recordSet {
-    # use the dummy store, this should only be used local
-    dummy = true
 
-    dynamo {
-      tableName = "recordSetTest"
-      provisionedReads=30
-      provisionedWrites=30
+  dynamodb.repositories {
+    record-set {
+      table-name = "recordSetTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
-  }
-  recordChange {
-    # use the dummy store, this should only be used local
-    dummy = true
-
-    dynamo {
-      tableName = "recordChangeTest"
-      provisionedReads = 30
-      provisionedWrites = 30
+    record-change {
+      table-name = "recordChangeTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    zone-change {
+      table-name = "zoneChangesTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    user {
+      table-name = "usersTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    group {
+      table-name = "groupsTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    group-change {
+      table-name = "groupChangesTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    membership {
+      table-name = "membershipTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
   }
 

modules/api/src/it/scala/vinyldns/api/engine/ZoneCommandHandlerIntegrationSpec.scala

@@ -23,13 +23,14 @@ import cats.implicits._
 import fs2.{Scheduler, Stream}
 import org.joda.time.DateTime
 import org.scalatest.concurrent.Eventually
+import org.scalatest.mockito.MockitoSugar
 import org.scalatest.time.{Millis, Seconds, Span}
 import vinyldns.api.{DynamoDBApiIntegrationSpec, VinylDNSTestData}
 import vinyldns.api.domain.record.RecordSetChangeGenerator
-import vinyldns.core.domain.batch.BatchChangeRepository
 import vinyldns.core.domain.record._
 import vinyldns.api.domain.zone._
 import vinyldns.api.engine.sqs.SqsConnection
+import vinyldns.api.repository.ApiDataAccessor
 import vinyldns.dynamodb.repository.{
   DynamoDBRecordChangeRepository,
   DynamoDBRecordSetRepository,
@@ -37,14 +38,21 @@ import vinyldns.dynamodb.repository.{
   DynamoDBZoneChangeRepository
 }
 import vinyldns.api.repository.mysql.TestMySqlInstance
+import vinyldns.core.domain.membership.{
+  GroupChangeRepository,
+  GroupRepository,
+  MembershipRepository,
+  UserRepository
+}
 import vinyldns.core.domain.zone._
 
 import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext}
+import scala.concurrent.ExecutionContext
 
 class ZoneCommandHandlerIntegrationSpec
     extends DynamoDBApiIntegrationSpec
     with VinylDNSTestData
+    with MockitoSugar
     with Eventually {
 
   import vinyldns.api.engine.sqs.SqsConverters._
@@ -65,11 +73,7 @@ class ZoneCommandHandlerIntegrationSpec
     PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis))
   private implicit val ec: ExecutionContext = scala.concurrent.ExecutionContext.global
 
-  private var recordChangeRepo: RecordChangeRepository = _
-  private var recordSetRepo: RecordSetRepository = _
-  private var zoneChangeRepo: ZoneChangeRepository = _
-  private var zoneRepo: ZoneRepository = _
-  private var batchChangeRepo: BatchChangeRepository = _
+  private var repositories: ApiDataAccessor = _
   private var sqsConn: SqsConnection = _
   private var str: Stream[IO, Unit] = _
   private val stopSignal = fs2.async.signalOf[IO, Boolean](false).unsafeRunSync()
@@ -119,38 +123,36 @@ class ZoneCommandHandlerIntegrationSpec
   }
 
   def setup(): Unit = {
-    val repos = (
-      DynamoDBRecordChangeRepository(recordChangeStoreConfig, dynamoIntegrationConfig),
+    val dynamoRepos = (
       DynamoDBRecordSetRepository(recordSetStoreConfig, dynamoIntegrationConfig),
+      DynamoDBRecordChangeRepository(recordChangeStoreConfig, dynamoIntegrationConfig),
       DynamoDBZoneChangeRepository(zoneChangeStoreConfig, dynamoIntegrationConfig)
     ).parTupled.unsafeRunSync()
 
-    recordChangeRepo = repos._1
-    recordSetRepo = repos._2
-    zoneChangeRepo = repos._3
-    zoneRepo = TestMySqlInstance.zoneRepository
-    batchChangeRepo = TestMySqlInstance.batchChangeRepository
+    repositories = ApiDataAccessor(
+      mock[UserRepository],
+      mock[GroupRepository],
+      mock[MembershipRepository],
+      mock[GroupChangeRepository],
+      dynamoRepos._1,
+      dynamoRepos._2,
+      dynamoRepos._3,
+      TestMySqlInstance.zoneRepository,
+      TestMySqlInstance.batchChangeRepository
+    )
+
     sqsConn = SqsConnection()
 
     //seed items database
-    waitForSuccess(zoneRepo.save(testZone))
-    waitForSuccess(recordChangeRepo.save(inDbRecordChange))
-    waitForSuccess(recordChangeRepo.save(inDbRecordChangeForSyncTest))
-    waitForSuccess(recordSetRepo.apply(inDbRecordChange))
-    waitForSuccess(recordSetRepo.apply(inDbRecordChangeForSyncTest))
-    waitForSuccess(zoneChangeRepo.save(inDbZoneChange))
-    // Run a noop query to make sure recordSetRepo is up
-    waitForSuccess(recordSetRepo.listRecordSets("1", None, None, None))
+    (
+      repositories.zoneRepository.save(testZone),
+      repositories.recordChangeRepository.save(inDbRecordChange),
+      repositories.recordChangeRepository.save(inDbRecordChangeForSyncTest),
+      repositories.recordSetRepository.apply(inDbRecordChange),
+      repositories.recordSetRepository.apply(inDbRecordChangeForSyncTest),
+      repositories.zoneChangeRepository.save(inDbZoneChange)).parTupled.unsafeRunSync()
 
-    str = ZoneCommandHandler.mainFlow(
-      zoneRepo,
-      zoneChangeRepo,
-      recordSetRepo,
-      recordChangeRepo,
-      batchChangeRepo,
-      sqsConn,
-      100.millis,
-      stopSignal)
+    str = ZoneCommandHandler.mainFlow(repositories, sqsConn, 100.millis, stopSignal)
     str.compile.drain.unsafeRunAsync { _ =>
       ()
     }
@@ -168,7 +170,7 @@ class ZoneCommandHandlerIntegrationSpec
 
       sendCommand(change, sqsConn).unsafeRunSync()
       eventually {
-        val getZone = zoneRepo.getZone(testZone.id).unsafeToFuture()
+        val getZone = repositories.zoneRepository.getZone(testZone.id).unsafeToFuture()
         whenReady(getZone) { zn =>
           zn.get.email shouldBe "updated@test.com"
         }
@@ -180,7 +182,9 @@ class ZoneCommandHandlerIntegrationSpec
         RecordSetChangeGenerator.forUpdate(inDbRecordSet, inDbRecordSet.copy(ttl = 1234), testZone)
       sendCommand(change, sqsConn).unsafeRunSync()
       eventually {
-        val getRs = recordSetRepo.getRecordSet(testZone.id, inDbRecordSet.id).unsafeToFuture()
+        val getRs = repositories.recordSetRepository
+          .getRecordSet(testZone.id, inDbRecordSet.id)
+          .unsafeToFuture()
         whenReady(getRs) { rs =>
           rs.get.ttl shouldBe 1234
         }
@@ -192,8 +196,9 @@ class ZoneCommandHandlerIntegrationSpec
       sendCommand(change, sqsConn).unsafeRunSync()
       eventually {
         val validatingQueries = for {
-          rs <- recordSetRepo.getRecordSet(testZone.id, inDbRecordSetForSyncTest.id)
-          ch <- recordChangeRepo.listRecordSetChanges(testZone.id)
+          rs <- repositories.recordSetRepository
+            .getRecordSet(testZone.id, inDbRecordSetForSyncTest.id)
+          ch <- repositories.recordChangeRepository.listRecordSetChanges(testZone.id)
         } yield (rs, ch)
 
         whenReady(validatingQueries.unsafeToFuture()) { data =>
@@ -210,9 +215,4 @@ class ZoneCommandHandlerIntegrationSpec
       }
     }
   }
-
-  private def waitForSuccess[T](f: => IO[T]): T = {
-    val waiting = f.unsafeToFuture().recover { case _ => Thread.sleep(2000); waitForSuccess(f) }
-    Await.result[T](waiting, 15.seconds)
-  }
 }

modules/api/src/it/scala/vinyldns/api/repository/mysql/TestMySqlInstance.scala

@@ -20,11 +20,15 @@ import vinyldns.api.VinylDNSConfig
 import vinyldns.api.crypto.Crypto
 import vinyldns.core.domain.batch.BatchChangeRepository
 import vinyldns.core.domain.zone.{ZoneChangeRepository, ZoneRepository}
-import vinyldns.core.repository.{DataStore, RepositoryName}
+import vinyldns.core.repository.{DataStore, DataStoreConfig, RepositoryName}
 
 object TestMySqlInstance {
+
+  lazy val mySqlConfig: DataStoreConfig =
+    pureconfig.loadConfigOrThrow[DataStoreConfig](VinylDNSConfig.vinyldnsConfig, "mysql")
+
   lazy val instance: DataStore =
-    new MySqlDataStoreProvider().load(VinylDNSConfig.mySqlConfig, Crypto.instance).unsafeRunSync()
+    new MySqlDataStoreProvider().load(mySqlConfig, Crypto.instance).unsafeRunSync()
 
   lazy val zoneRepository: ZoneRepository =
     instance.get[ZoneRepository](RepositoryName.zone).get

modules/api/src/main/resources/application.conf

@@ -46,6 +46,44 @@ vinyldns {
     }
   }
 
+  dynamodb.repositories {
+    record-set {
+      table-name = "recordSetTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    record-change {
+      table-name = "recordChangeTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    zone-change {
+      table-name = "zoneChangesTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    user {
+      table-name = "usersTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    group {
+      table-name = "groupsTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    group-change {
+      table-name = "groupChangesTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+    membership {
+      table-name = "membershipTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
+    }
+  }
+
   sync-delay = 10000 # 10 second delay for resyncing zone
 
   batch-change-limit = 20 # Max change limit per batch request

modules/api/src/main/resources/reference.conf

@@ -33,7 +33,7 @@ vinyldns {
     port = 9000
   }
 
-  data-stores = ["mysql"]
+  data-stores = ["mysql", "dynamodb"]
 
   mysql {
     class-name = "vinyldns.api.repository.mysql.MySqlDataStoreProvider"
@@ -59,60 +59,18 @@ vinyldns {
     }
   }
 
-  dynamo {
-    key = "vinyldnsTest"
-    secret = "notNeededForDynamoDbLocal"
-    endpoint = "http://127.0.0.1:19000"
-    region = "us-east-1" # note: we are always in us-east-1, but this can be overridden
-  }
+  dynamodb {
+    class-name = "vinyldns.dynamodb.repository.DynamoDBDataStoreProvider"
 
-  zoneChanges {
-    dynamo {
-      tableName = "zoneChanges"
-      provisionedReads=30
-      provisionedWrites=30
+    settings {
+      key = "vinyldnsTest"
+      secret = "notNeededForDynamoDbLocal"
+      endpoint = "http://127.0.0.1:19000"
+      region = "us-east-1" # note: we are always in us-east-1, but this can be overridden
     }
-  }
-  recordSet {
-    dynamo {
-      tableName = "recordSet"
-      provisionedReads=30
-      provisionedWrites=30
-    }
-  }
-  recordChange {
-    dynamo {
-      tableName = "recordChange"
-      provisionedReads=30
-      provisionedWrites=30
-    }
-  }
-  users {
-    dynamo {
-      tableName = "users"
-      provisionedReads=30
-      provisionedWrites=30
-    }
-  }
-  groups {
-    dynamo {
-      tableName = "groups"
-      provisionedReads=30
-      provisionedWrites=30
-    }
-  }
-  groupChanges {
-    dynamo {
-      tableName = "groupChanges"
-      provisionedReads=30
-      provisionedWrites=30
-    }
-  }
-  membership {
-    dynamo {
-      tableName = "membership"
-      provisionedReads=30
-      provisionedWrites=30
+
+    repositories {
+      # override
     }
   }
 

modules/api/src/main/scala/vinyldns/api/Boot.scala

@@ -32,14 +32,10 @@ import vinyldns.api.domain.record.RecordSetService
 import vinyldns.api.domain.zone._
 import vinyldns.api.engine.ProductionZoneCommandHandler
 import vinyldns.api.engine.sqs.{SqsCommandBus, SqsConnection}
-import vinyldns.dynamodb.repository._
-import vinyldns.api.repository.TestDataLoader
-import vinyldns.api.repository.mysql.MySqlDataStoreProvider
+import vinyldns.api.repository.{ApiDataAccessor, ApiDataAccessorProvider, TestDataLoader}
 import vinyldns.api.route.{HealthService, VinylDNSService}
 import vinyldns.core.VinylDNSMetrics
-import vinyldns.core.domain.batch.BatchChangeRepository
-import vinyldns.core.domain.zone.ZoneRepository
-import vinyldns.core.repository.{DataStoreStartupError, RepositoryName}
+import vinyldns.core.repository.DataStoreLoader
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.io.{Codec, Source}
@@ -65,41 +61,10 @@ object Boot extends App {
     for {
       banner <- vinyldnsBanner()
       crypto <- IO(Crypto.instance) // load crypto
-      // TODO datastore loading will not be hardcoded by type here
-      mySqlDataStore <- new MySqlDataStoreProvider().load(VinylDNSConfig.mySqlConfig, crypto)
-      zoneRepo <- IO.fromEither(
-        mySqlDataStore
-          .get[ZoneRepository](RepositoryName.zone)
-          .toRight[Throwable](DataStoreStartupError("Missing zone repository")))
-      batchChangeRepo <- IO.fromEither(
-        mySqlDataStore
-          .get[BatchChangeRepository](RepositoryName.batchChange)
-          .toRight[Throwable](DataStoreStartupError("Missing zone repository")))
-      // TODO this also will all be removed with dynamic loading
-      userRepo <- DynamoDBUserRepository(
-        VinylDNSConfig.usersStoreConfig,
-        VinylDNSConfig.dynamoConfig,
-        crypto
-      )
-      groupRepo <- DynamoDBGroupRepository(
-        VinylDNSConfig.groupsStoreConfig,
-        VinylDNSConfig.dynamoConfig)
-      membershipRepo <- DynamoDBMembershipRepository(
-        VinylDNSConfig.membershipStoreConfig,
-        VinylDNSConfig.dynamoConfig)
-      groupChangeRepo <- DynamoDBGroupChangeRepository(
-        VinylDNSConfig.groupChangesStoreConfig,
-        VinylDNSConfig.dynamoConfig)
-      recordSetRepo <- DynamoDBRecordSetRepository(
-        VinylDNSConfig.recordSetStoreConfig,
-        VinylDNSConfig.dynamoConfig)
-      recordChangeRepo <- DynamoDBRecordChangeRepository(
-        VinylDNSConfig.recordChangeStoreConfig,
-        VinylDNSConfig.dynamoConfig)
-      zoneChangeRepo <- DynamoDBZoneChangeRepository(
-        VinylDNSConfig.zoneChangeStoreConfig,
-        VinylDNSConfig.dynamoConfig)
-      _ <- TestDataLoader.loadTestData(userRepo)
+      repoConfigs <- VinylDNSConfig.dataStoreConfigs
+      repositories <- DataStoreLoader
+        .loadAll[ApiDataAccessor](repoConfigs, crypto, ApiDataAccessorProvider)
+      _ <- TestDataLoader.loadTestData(repositories.userRepository)
       sqsConfig <- IO(VinylDNSConfig.sqsConfig)
       sqsConnection <- IO(SqsConnection(sqsConfig))
       processingDisabled <- IO(VinylDNSConfig.vinyldnsConfig.getBoolean("processing-disabled"))
@@ -109,47 +74,26 @@ object Boot extends App {
       batchChangeLimit <- IO(VinylDNSConfig.vinyldnsConfig.getInt("batch-change-limit"))
       syncDelay <- IO(VinylDNSConfig.vinyldnsConfig.getInt("sync-delay"))
       _ <- fs2.async.start(
-        ProductionZoneCommandHandler.run(
-          sqsConnection,
-          processingSignal,
-          zoneRepo,
-          zoneChangeRepo,
-          recordChangeRepo,
-          recordSetRepo,
-          batchChangeRepo,
-          sqsConfig))
+        ProductionZoneCommandHandler.run(sqsConnection, processingSignal, repositories, sqsConfig))
     } yield {
       val zoneValidations = new ZoneValidations(syncDelay)
       val batchChangeValidations = new BatchChangeValidations(batchChangeLimit, AccessValidations)
       val commandBus = new SqsCommandBus(sqsConnection)
-      val membershipService =
-        new MembershipService(groupRepo, userRepo, membershipRepo, zoneRepo, groupChangeRepo)
+      val membershipService = MembershipService(repositories)
       val connectionValidator =
         new ZoneConnectionValidator(VinylDNSConfig.defaultZoneConnection)
-      val recordSetService = new RecordSetService(
-        zoneRepo,
-        recordSetRepo,
-        recordChangeRepo,
-        userRepo,
-        commandBus,
-        AccessValidations)
-      val zoneService = new ZoneService(
-        zoneRepo,
-        groupRepo,
-        userRepo,
-        zoneChangeRepo,
+      val recordSetService = RecordSetService(repositories, commandBus, AccessValidations)
+      val zoneService = ZoneService(
+        repositories,
         connectionValidator,
         commandBus,
         zoneValidations,
         AccessValidations)
-      val healthService = new HealthService(zoneRepo)
-      val batchChangeConverter = new BatchChangeConverter(batchChangeRepo, commandBus)
-      val batchChangeService = new BatchChangeService(
-        zoneRepo,
-        recordSetRepo,
-        batchChangeValidations,
-        batchChangeRepo,
-        batchChangeConverter)
+      val healthService = new HealthService(repositories.zoneRepository)
+      val batchChangeConverter =
+        new BatchChangeConverter(repositories.batchChangeRepository, commandBus)
+      val batchChangeService =
+        BatchChangeService(repositories, batchChangeValidations, batchChangeConverter)
       val collectorRegistry = CollectorRegistry.defaultRegistry
       val vinyldnsService = new VinylDNSService(
         membershipService,
@@ -158,7 +102,10 @@ object Boot extends App {
         healthService,
         recordSetService,
         batchChangeService,
-        collectorRegistry)
+        collectorRegistry,
+        repositories.userRepository,
+        repositories.membershipRepository
+      )
 
       DefaultExports.initialize()
       collectorRegistry.register(new DropwizardExports(VinylDNSMetrics.metricsRegistry))

modules/api/src/main/scala/vinyldns/api/VinylDNSConfig.scala

@@ -17,7 +17,10 @@
 package vinyldns.api
 
 import akka.actor.ActorSystem
+import cats.effect.IO
+import cats.implicits._
 import com.typesafe.config.{Config, ConfigFactory}
+import pureconfig.module.catseffect.loadConfigF
 import pureconfig.{CamelCase, ConfigFieldMapping, ProductHint}
 import vinyldns.api.VinylDNSConfig.vinyldnsConfig
 import vinyldns.api.crypto.Crypto
@@ -33,21 +36,18 @@ object VinylDNSConfig {
   lazy val config: Config = ConfigFactory.load()
   lazy val vinyldnsConfig: Config = config.getConfig("vinyldns")
 
-  lazy val dynamoConfig = DynamoConfig.dynamoConfig
-  lazy val zoneChangeStoreConfig: DynamoDBRepositorySettings = DynamoConfig.zoneChangeStoreConfig
-  lazy val recordSetStoreConfig: DynamoDBRepositorySettings = DynamoConfig.recordSetStoreConfig
-  lazy val recordChangeStoreConfig: DynamoDBRepositorySettings =
-    DynamoConfig.recordChangeStoreConfig
-  lazy val usersStoreConfig: DynamoDBRepositorySettings = DynamoConfig.usersStoreConfig
-  lazy val groupsStoreConfig: DynamoDBRepositorySettings = DynamoConfig.groupsStoreConfig
-  lazy val groupChangesStoreConfig: DynamoDBRepositorySettings =
-    DynamoConfig.groupChangesStoreConfig
-  lazy val membershipStoreConfig: DynamoDBRepositorySettings = DynamoConfig.membershipStoreConfig
+  lazy val dataStoreConfigs: IO[List[DataStoreConfig]] =
+    vinyldnsConfig
+      .getStringList("data-stores")
+      .asScala
+      .toList
+      .map { configKey =>
+        loadConfigF[IO, DataStoreConfig](vinyldnsConfig, configKey)
+      }
+      .parSequence
 
   lazy val restConfig: Config = vinyldnsConfig.getConfig("rest")
   lazy val monitoringConfig: Config = vinyldnsConfig.getConfig("monitoring")
-  lazy val mySqlConfig: DataStoreConfig =
-    pureconfig.loadConfigOrThrow[DataStoreConfig](vinyldnsConfig, "mysql")
   lazy val sqsConfig: Config = vinyldnsConfig.getConfig("sqs")
   lazy val cryptoConfig: Config = vinyldnsConfig.getConfig("crypto")
   lazy val system: ActorSystem = ActorSystem("VinylDNS", VinylDNSConfig.config)

modules/api/src/main/scala/vinyldns/api/domain/auth/AuthPrincipalProvider.scala

@@ -17,10 +17,7 @@
 package vinyldns.api.domain.auth
 
 import cats.effect._
-import vinyldns.api.VinylDNSConfig
-import vinyldns.api.crypto.Crypto
 import vinyldns.core.domain.membership.{MembershipRepository, User, UserRepository}
-import vinyldns.dynamodb.repository.{DynamoDBMembershipRepository, DynamoDBUserRepository}
 import vinyldns.core.domain.auth.AuthPrincipal
 import vinyldns.core.route.Monitored
 
@@ -52,19 +49,3 @@ class MembershipAuthPrincipalProvider(
       membershipRepo.getGroupsForUser(userId)
     }
 }
-
-object MembershipAuthPrincipalProvider {
-  // TODO this has to be dynamic!!!
-  val userRepository: UserRepository =
-    DynamoDBUserRepository(
-      VinylDNSConfig.usersStoreConfig,
-      VinylDNSConfig.dynamoConfig,
-      Crypto.instance)
-      .unsafeRunSync()
-  val membershipRepository: MembershipRepository =
-    DynamoDBMembershipRepository(VinylDNSConfig.membershipStoreConfig, VinylDNSConfig.dynamoConfig)
-      .unsafeRunSync()
-
-  def apply(): MembershipAuthPrincipalProvider =
-    new MembershipAuthPrincipalProvider(userRepository, membershipRepository)
-}

modules/api/src/main/scala/vinyldns/api/domain/batch/BatchChangeService.scala

@@ -29,8 +29,22 @@ import vinyldns.core.domain.record.RecordType._
 import vinyldns.core.domain.record.{RecordSet, RecordSetRepository}
 import vinyldns.core.domain.zone.ZoneRepository
 import vinyldns.api.domain.{RecordAlreadyExists, ZoneDiscoveryError}
+import vinyldns.api.repository.ApiDataAccessor
 import vinyldns.core.domain.batch.{BatchChange, BatchChangeRepository, BatchChangeSummaryList}
 
+object BatchChangeService {
+  def apply(
+      dataAccessor: ApiDataAccessor,
+      batchChangeValidations: BatchChangeValidationsAlgebra,
+      batchChangeConverter: BatchChangeConverterAlgebra): BatchChangeService =
+    new BatchChangeService(
+      dataAccessor.zoneRepository,
+      dataAccessor.recordSetRepository,
+      batchChangeValidations,
+      dataAccessor.batchChangeRepository,
+      batchChangeConverter)
+}
+
 class BatchChangeService(
     zoneRepository: ZoneRepository,
     recordSetRepository: RecordSetRepository,

modules/api/src/main/scala/vinyldns/api/domain/membership/MembershipService.scala

@@ -18,11 +18,23 @@ package vinyldns.api.domain.membership
 
 import cats.implicits._
 import vinyldns.api.Interfaces._
+import vinyldns.api.repository.ApiDataAccessor
 import vinyldns.core.domain.auth.AuthPrincipal
 import vinyldns.core.domain.membership.LockStatus.LockStatus
 import vinyldns.core.domain.zone.ZoneRepository
 import vinyldns.core.domain.membership._
 
+object MembershipService {
+  def apply(dataAccessor: ApiDataAccessor): MembershipService =
+    new MembershipService(
+      dataAccessor.groupRepository,
+      dataAccessor.userRepository,
+      dataAccessor.membershipRepository,
+      dataAccessor.zoneRepository,
+      dataAccessor.groupChangeRepository
+    )
+}
+
 class MembershipService(
     groupRepo: GroupRepository,
     userRepo: UserRepository,

modules/api/src/main/scala/vinyldns/api/domain/record/RecordSetService.scala

@@ -22,10 +22,26 @@ import vinyldns.core.domain.auth.AuthPrincipal
 import vinyldns.api.domain.engine.EngineCommandBus
 import vinyldns.core.domain.membership.{User, UserRepository}
 import vinyldns.api.domain.zone._
+import vinyldns.api.repository.ApiDataAccessor
 import vinyldns.api.route.ListRecordSetsResponse
 import vinyldns.core.domain.record._
 import vinyldns.core.domain.zone.{Zone, ZoneCommandResult, ZoneRepository}
 
+object RecordSetService {
+  def apply(
+      dataAccessor: ApiDataAccessor,
+      commandBus: EngineCommandBus,
+      accessValidation: AccessValidationAlgebra): RecordSetService =
+    new RecordSetService(
+      dataAccessor.zoneRepository,
+      dataAccessor.recordSetRepository,
+      dataAccessor.recordChangeRepository,
+      dataAccessor.userRepository,
+      commandBus,
+      accessValidation
+    )
+}
+
 class RecordSetService(
     zoneRepository: ZoneRepository,
     recordSetRepository: RecordSetRepository,

modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneService.scala

@@ -21,9 +21,29 @@ import vinyldns.api.Interfaces._
 import vinyldns.api.domain.AccessValidationAlgebra
 import vinyldns.core.domain.auth.AuthPrincipal
 import vinyldns.api.domain.engine.EngineCommandBus
+import vinyldns.api.repository.ApiDataAccessor
 import vinyldns.core.domain.membership.{Group, GroupRepository, User, UserRepository}
 import vinyldns.core.domain.zone._
 
+object ZoneService {
+  def apply(
+      dataAccessor: ApiDataAccessor,
+      connectionValidator: ZoneConnectionValidatorAlgebra,
+      commandBus: EngineCommandBus,
+      zoneValidations: ZoneValidations,
+      accessValidation: AccessValidationAlgebra): ZoneService =
+    new ZoneService(
+      dataAccessor.zoneRepository,
+      dataAccessor.groupRepository,
+      dataAccessor.userRepository,
+      dataAccessor.zoneChangeRepository,
+      connectionValidator,
+      commandBus,
+      zoneValidations,
+      accessValidation
+    )
+}
+
 class ZoneService(
     zoneRepository: ZoneRepository,
     groupRepository: GroupRepository,

modules/api/src/main/scala/vinyldns/api/engine/ZoneCommandHandler.scala

@@ -26,10 +26,10 @@ import fs2.async.mutable.Signal
 import org.slf4j.LoggerFactory
 import vinyldns.api.VinylDNSConfig
 import vinyldns.api.domain.dns.DnsConnection
-import vinyldns.core.domain.record.{RecordChangeRepository, RecordSetChange, RecordSetRepository}
-import vinyldns.core.domain.zone.{ZoneChange, ZoneChangeRepository, ZoneChangeType, ZoneRepository}
+import vinyldns.core.domain.record.RecordSetChange
+import vinyldns.core.domain.zone.{ZoneChange, ZoneChangeType}
 import vinyldns.api.engine.sqs.SqsConnection
-import vinyldns.core.domain.batch.BatchChangeRepository
+import vinyldns.api.repository.ApiDataAccessor
 
 import scala.collection.JavaConverters._
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -63,11 +63,7 @@ object ZoneCommandHandler {
       extends ChangeRequest
 
   def mainFlow(
-      zoneRepository: ZoneRepository,
-      zoneChangeRepository: ZoneChangeRepository,
-      recordSetRepository: RecordSetRepository,
-      recordChangeRepository: RecordChangeRepository,
-      batchChangeRepository: BatchChangeRepository,
+      dataAccessor: ApiDataAccessor,
       sqsConnection: SqsConnection,
       pollingInterval: FiniteDuration,
       pauseSignal: Signal[IO, Boolean])(implicit scheduler: Scheduler): Stream[IO, Unit] = {
@@ -79,10 +75,15 @@ object ZoneCommandHandler {
     val increaseTimeoutForZoneSyncs = changeVisibilityTimeoutForZoneSyncs(sqsConnection)
 
     // Handlers for each type of change request
-    val zoneChangeHandler = ZoneChangeHandler(zoneRepository, zoneChangeRepository)
+    val zoneChangeHandler =
+      ZoneChangeHandler(dataAccessor.zoneRepository, dataAccessor.zoneChangeRepository)
     val recordChangeHandler =
-      RecordSetChangeHandler(recordSetRepository, recordChangeRepository, batchChangeRepository)
-    val zoneSyncHandler = ZoneSyncHandler(recordSetRepository, recordChangeRepository)
+      RecordSetChangeHandler(
+        dataAccessor.recordSetRepository,
+        dataAccessor.recordChangeRepository,
+        dataAccessor.batchChangeRepository)
+    val zoneSyncHandler =
+      ZoneSyncHandler(dataAccessor.recordSetRepository, dataAccessor.recordChangeRepository)
 
     val changeRequestProcessor =
       processChangeRequests(zoneChangeHandler, recordChangeHandler, zoneSyncHandler)
@@ -237,11 +238,7 @@ object ProductionZoneCommandHandler {
   def run(
       sqsConnection: SqsConnection,
       processingSignal: Signal[IO, Boolean],
-      zoneRepository: ZoneRepository,
-      zoneChangeRepository: ZoneChangeRepository,
-      recordChangeRepository: RecordChangeRepository,
-      recordSetRepository: RecordSetRepository,
-      batchChangeRepository: BatchChangeRepository,
+      dataAccessor: ApiDataAccessor,
       config: Config): IO[Unit] = {
     implicit val scheduler: Scheduler =
       Scheduler.fromScheduledExecutorService(Executors.newScheduledThreadPool(2))
@@ -250,15 +247,7 @@ object ProductionZoneCommandHandler {
       pollingInterval <- IO.pure(
         config.getDuration("polling-interval", TimeUnit.MILLISECONDS).milliseconds)
       flow <- ZoneCommandHandler
-        .mainFlow(
-          zoneRepository,
-          zoneChangeRepository,
-          recordSetRepository,
-          recordChangeRepository,
-          batchChangeRepository,
-          sqsConnection,
-          pollingInterval,
-          processingSignal)
+        .mainFlow(dataAccessor, sqsConnection, pollingInterval, processingSignal)
         .compile
         .drain
     } yield flow

modules/api/src/main/scala/vinyldns/api/route/VinylDNSAuthentication.scala

@@ -21,7 +21,7 @@ import akka.http.scaladsl.server.RequestContext
 import cats.effect._
 import cats.syntax.all._
 import vinyldns.api.crypto.Crypto
-import vinyldns.api.domain.auth.{AuthPrincipalProvider, MembershipAuthPrincipalProvider}
+import vinyldns.api.domain.auth.AuthPrincipalProvider
 import vinyldns.core.crypto.CryptoAlgebra
 import vinyldns.core.domain.auth.AuthPrincipal
 import vinyldns.core.domain.membership.LockStatus
@@ -34,9 +34,27 @@ final case class AuthMissing(msg: String) extends VinylDNSAuthenticationError(ms
 final case class AuthRejected(reason: String) extends VinylDNSAuthenticationError(reason)
 final case class AccountLocked(reason: String) extends VinylDNSAuthenticationError(reason)
 
-trait VinylDNSAuthentication extends Monitored {
-  val authenticator: Aws4Authenticator
-  val authPrincipalProvider: AuthPrincipalProvider
+trait VinylDNSAuthenticator {
+  def authenticate(
+      ctx: RequestContext,
+      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]]
+}
+
+class ProductionVinylDNSAuthenticator(
+    val authenticator: Aws4Authenticator,
+    val authPrincipalProvider: AuthPrincipalProvider)
+    extends VinylDNSAuthenticator
+    with Monitored {
+
+  def authenticate(
+      ctx: RequestContext,
+      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] =
+    // Need to refactor getAuthPrincipal to be an IO[Either[E, A]] instead of how it is implemented.
+    getAuthPrincipal(ctx, content).attempt.flatMap {
+      case Left(e: VinylDNSAuthenticationError) => IO.pure(Left(e))
+      case Right(ok) => IO.pure(Right(ok))
+      case Left(e) => IO.raiseError(e)
+    }
 
   /**
     * Gets the auth header from the request.  If the auth header is not found then the
@@ -111,7 +129,7 @@ trait VinylDNSAuthentication extends Monitored {
     * @param ctx The Http Request Context
     * @return A Future containing the AuthPrincipal for the request.
     */
-  def authenticate(ctx: RequestContext, content: String): IO[AuthPrincipal] =
+  def getAuthPrincipal(ctx: RequestContext, content: String): IO[AuthPrincipal] =
     for {
       authHeader <- getAuthHeader(ctx)
       regexMatch <- parseAuthHeader(authHeader)
@@ -138,30 +156,3 @@ trait VinylDNSAuthentication extends Monitored {
         IO.raiseError(AuthRejected(s"Account with accessKey $accessKey specified was not found"))
     }
 }
-
-class VinylDNSAuthenticator(
-    val authenticator: Aws4Authenticator,
-    val authPrincipalProvider: AuthPrincipalProvider)
-    extends VinylDNSAuthentication {
-
-  def apply(
-      ctx: RequestContext,
-      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] =
-    // Need to refactor authenticate to be an IO[Either[E, A]] instead of how it is implemented, for the time being...
-    authenticate(ctx, content).attempt.flatMap {
-      case Left(e: VinylDNSAuthenticationError) => IO.pure(Left(e))
-      case Right(ok) => IO.pure(Right(ok))
-      case Left(e) => IO.raiseError(e)
-    }
-}
-
-object VinylDNSAuthenticator {
-  lazy val aws4Authenticator = new Aws4Authenticator
-  lazy val authPrincipalProvider = MembershipAuthPrincipalProvider()
-  lazy val authenticator = new VinylDNSAuthenticator(aws4Authenticator, authPrincipalProvider)
-
-  def apply(
-      ctx: RequestContext,
-      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] =
-    authenticator.apply(ctx, content)
-}

modules/api/src/main/scala/vinyldns/api/route/VinylDNSDirectives.scala

@@ -22,7 +22,6 @@ import akka.http.scaladsl.server._
 import akka.http.scaladsl.server.directives.BasicDirectives
 import cats.data.Validated.{Invalid, Valid}
 import cats.data.ValidatedNel
-import cats.effect._
 import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 import vinyldns.core.domain.auth.AuthPrincipal
@@ -34,21 +33,14 @@ import scala.util.control.NonFatal
 
 trait VinylDNSDirectives extends Directives {
 
-  /**
-    * Authenticator that takes a request context and yields an Authentication, which is an Either
-    * that holds a Left - Rejection, or Right - AuthPrincipal.
-    * @return an Authentication with the AuthPrincipal as looked up from the request, or a Left(Rejection)
-    */
-  def vinyldnsAuthenticator(
-      ctx: RequestContext,
-      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] =
-    VinylDNSAuthenticator(ctx, content)
+  val vinylDNSAuthenticator: VinylDNSAuthenticator
 
   def authenticate: Directive1[AuthPrincipal] =
     extractExecutionContext.flatMap { implicit ec ⇒
       extractRequestContext.flatMap { ctx =>
         extractStrictEntity(10.seconds).flatMap { strictEntity =>
-          onSuccess(vinyldnsAuthenticator(ctx, strictEntity.data.utf8String).unsafeToFuture())
+          onSuccess(
+            vinylDNSAuthenticator.authenticate(ctx, strictEntity.data.utf8String).unsafeToFuture())
             .flatMap {
               case Right(authPrincipal) ⇒
                 provide(authPrincipal)

modules/api/src/main/scala/vinyldns/api/route/VinylDNSService.scala

@@ -25,10 +25,12 @@ import akka.http.scaladsl.server.directives.LogEntry
 import cats.effect.IO
 import fs2.async.mutable.Signal
 import io.prometheus.client.CollectorRegistry
+import vinyldns.api.domain.auth.MembershipAuthPrincipalProvider
 import vinyldns.api.domain.batch.BatchChangeServiceAlgebra
 import vinyldns.api.domain.membership.MembershipServiceAlgebra
 import vinyldns.api.domain.record.RecordSetServiceAlgebra
 import vinyldns.api.domain.zone.ZoneServiceAlgebra
+import vinyldns.core.domain.membership.{MembershipRepository, UserRepository}
 
 import scala.util.matching.Regex
 
@@ -103,7 +105,9 @@ class VinylDNSService(
     val healthService: HealthService,
     val recordSetService: RecordSetServiceAlgebra,
     val batchChangeService: BatchChangeServiceAlgebra,
-    val collectorRegistry: CollectorRegistry)
+    val collectorRegistry: CollectorRegistry,
+    userRepository: UserRepository,
+    membershipRepository: MembershipRepository)
     extends VinylDNSDirectives
     with PingRoute
     with ZoneRoute
@@ -117,6 +121,12 @@ class VinylDNSService(
     with VinylDNSJsonProtocol
     with JsonValidationRejection {
 
+  val aws4Authenticator = new Aws4Authenticator
+  val authPrincipalProvider =
+    new MembershipAuthPrincipalProvider(userRepository, membershipRepository)
+  val vinylDNSAuthenticator: VinylDNSAuthenticator =
+    new ProductionVinylDNSAuthenticator(aws4Authenticator, authPrincipalProvider)
+
   // Authenticated routes must go first
   def authenticatedRoutes: server.Route =
     handleRejections(validationRejectionHandler)(authenticate { authPrincipal =>

modules/api/src/test/resources/application.conf

@@ -30,67 +30,41 @@ vinyldns {
     }
   }
 
-  accounts {
-    dummy = true
-
-    dynamo {
-      key = "dynamoKey"
-      secret = "dynamoSecret"
-      endpoint = "dynamoEndpoint"
+  dynamodb.repositories {
+    record-set {
+      table-name = "recordSetTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
-  }
-
-  dynamo {
-    key="dynamoKey"
-    secret="dynamoSecret"
-    endpoint="dynamoEndpoint"
-  }
-
-  zoneChanges {
-    dynamo {
-      tableName = "zoneChanges"
-      provisionedReads=40
-      provisionedWrites=30
+    record-change {
+      table-name = "recordChangeTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
-  }
-
-  recordSet {
-    dynamo {
-      tableName = "recordSet"
-      provisionedReads=40
-      provisionedWrites=30
+    zone-change {
+      table-name = "zoneChangesTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
-  }
-
-  recordChange {
-    dynamo {
-      tableName = "recordChange"
-      provisionedReads=40
-      provisionedWrites=30
+    user {
+      table-name = "usersTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
-  }
-
-  groups {
-    dynamo {
-      tableName = "groups"
-      provisionedReads=40
-      provisionedWrites=30
+    group {
+      table-name = "groupsTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
-  }
-
-  groupChanges {
-    dynamo {
-      tableName = "groupChanges"
-      provisionedReads=40
-      provisionedWrites=30
+    group-change {
+      table-name = "groupChangesTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
-  }
-
-  membership {
-    dynamo {
-      tableName = "membership"
-      provisionedReads=40
-      provisionedWrites=30
+    membership {
+      table-name = "membershipTest"
+      provisioned-reads = 30
+      provisioned-writes = 20
     }
   }
 

modules/api/src/test/scala/vinyldns/api/VinylDNSConfigSpec.scala

@@ -17,6 +17,7 @@
 package vinyldns.api
 
 import org.scalatest.{Matchers, WordSpec}
+import vinyldns.core.repository.RepositoryName._
 
 class VinylDNSConfigSpec extends WordSpec with Matchers {
 
@@ -26,46 +27,26 @@ class VinylDNSConfigSpec extends WordSpec with Matchers {
       restConfig.getInt("port") shouldBe 9000
     }
 
-    "load the dynamo config" in {
-      val dynamoConfig = VinylDNSConfig.dynamoConfig
-      dynamoConfig.key shouldBe "dynamoKey"
-      dynamoConfig.secret shouldBe "dynamoSecret"
-      dynamoConfig.endpoint shouldBe "dynamoEndpoint"
-    }
+    "properly load the datastore configs" in {
 
-    "load the zone change repository config" in {
-      val config = VinylDNSConfig.zoneChangeStoreConfig
-      config.tableName shouldBe "zoneChanges"
-      config.provisionedReads shouldBe 40
-      config.provisionedWrites shouldBe 30
+      VinylDNSConfig.dataStoreConfigs.unsafeRunSync.length shouldBe 2
     }
+    "assign the correct mysql repositories" in {
+      val mysqlConfig =
+        VinylDNSConfig.dataStoreConfigs.unsafeRunSync
+          .find(_.className == "vinyldns.api.repository.mysql.MySqlDataStoreProvider")
+          .get
 
-    "load the record change repository config" in {
-      val config = VinylDNSConfig.recordChangeStoreConfig
-      config.tableName shouldBe "recordChange"
-      config.provisionedReads shouldBe 40
-      config.provisionedWrites shouldBe 30
+      mysqlConfig.repositories.keys should contain theSameElementsAs Set(zone, batchChange)
     }
+    "assign the correct dynamodb repositories" in {
+      val dynamodbConfig =
+        VinylDNSConfig.dataStoreConfigs.unsafeRunSync
+          .find(_.className == "vinyldns.dynamodb.repository.DynamoDBDataStoreProvider")
+          .get
 
-    "load the membership repository config" in {
-      val config = VinylDNSConfig.membershipStoreConfig
-      config.tableName shouldBe "membership"
-      config.provisionedReads shouldBe 40
-      config.provisionedWrites shouldBe 30
-    }
-
-    "load the record set repository config" in {
-      val config = VinylDNSConfig.recordSetStoreConfig
-      config.tableName shouldBe "recordSet"
-      config.provisionedReads shouldBe 40
-      config.provisionedWrites shouldBe 30
-    }
-
-    "load the group repository config" in {
-      val config = VinylDNSConfig.groupsStoreConfig
-      config.tableName shouldBe "groups"
-      config.provisionedReads shouldBe 40
-      config.provisionedWrites shouldBe 30
+      dynamodbConfig.repositories.keys should contain theSameElementsAs
+        Set(user, group, membership, groupChange, recordSet, recordChange, zoneChange)
     }
   }
 }

modules/api/src/test/scala/vinyldns/api/route/BatchChangeRoutingSpec.scala

@@ -47,6 +47,7 @@ class BatchChangeRoutingSpec
     with GroupTestData {
 
   val batchChangeService: BatchChangeServiceAlgebra = TestBatchChangeService
+  val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator(okUserAuth)
 
   import vinyldns.core.domain.batch.SingleChangeStatus._
 

modules/api/src/test/scala/vinyldns/api/route/HealthCheckRoutingSpec.scala

@@ -25,7 +25,6 @@ import org.mockito.Mockito.doReturn
 import org.scalatest.mockito.MockitoSugar
 import org.scalatest.{Matchers, OneInstancePerTest, WordSpec}
 import vinyldns.core.domain.zone.ZoneRepository
-
 import cats.effect._
 
 class HealthCheckRoutingSpec
@@ -34,7 +33,6 @@ class HealthCheckRoutingSpec
     with Directives
     with HealthCheckRoute
     with VinylDNSJsonProtocol
-    with VinylDNSDirectives
     with OneInstancePerTest
     with Matchers
     with MockitoSugar {

modules/api/src/test/scala/vinyldns/api/route/MembershipRoutingSpec.scala

@@ -51,6 +51,7 @@ class MembershipRoutingSpec
     with BeforeAndAfterEach {
 
   val membershipService: MembershipService = mock[MembershipService]
+  val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator(okAuth)
 
   override protected def beforeEach(): Unit = reset(membershipService)
 

modules/api/src/test/scala/vinyldns/api/route/RecordSetRoutingSpec.scala

@@ -17,9 +17,8 @@
 package vinyldns.api.route
 
 import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest, StatusCodes}
-import akka.http.scaladsl.server.{Directives, RequestContext, Route}
+import akka.http.scaladsl.server.{Directives, Route}
 import akka.http.scaladsl.testkit.ScalatestRouteTest
-import cats.effect._
 import org.joda.time.DateTime
 import org.json4s.JsonDSL._
 import org.json4s._
@@ -482,10 +481,7 @@ class RecordSetRoutingSpec
 
   val recordSetService: RecordSetServiceAlgebra = new TestService
 
-  override def vinyldnsAuthenticator(
-      ctx: RequestContext,
-      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] =
-    IO.pure(Right(okAuth))
+  val vinylDNSAuthenticator = new TestVinylDNSAuthenticator(okAuth)
 
   private def rsJson(recordSet: RecordSet): String =
     compact(render(Extraction.decompose(recordSet)))

modules/api/src/test/scala/vinyldns/api/route/TestVinylDNSAuthenticator.scala

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 Comcast Cable Communications Management, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vinyldns.api.route
+
+import akka.http.scaladsl.server.RequestContext
+import cats.effect.IO
+import vinyldns.core.domain.auth.AuthPrincipal
+
+class TestVinylDNSAuthenticator(authPrincipal: AuthPrincipal) extends VinylDNSAuthenticator {
+
+  def authenticate(
+      ctx: RequestContext,
+      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] =
+    IO.pure(Right(authPrincipal))
+}

modules/api/src/test/scala/vinyldns/api/route/VinylDNSAuthenticatorSpec.scala

@@ -24,7 +24,7 @@ import org.mockito.Mockito._
 import org.scalatest.mockito.MockitoSugar
 import org.scalatest.{Matchers, WordSpec}
 import vinyldns.api.domain.auth.AuthPrincipalProvider
-import vinyldns.api.{GroupTestData}
+import vinyldns.api.GroupTestData
 import vinyldns.core.crypto.CryptoAlgebra
 
 class VinylDNSAuthenticatorSpec
@@ -35,7 +35,8 @@ class VinylDNSAuthenticatorSpec
   private val mockAuthenticator = mock[Aws4Authenticator]
   private val mockAuthPrincipalProvider = mock[AuthPrincipalProvider]
 
-  private val underTest = new VinylDNSAuthenticator(mockAuthenticator, mockAuthPrincipalProvider)
+  private val underTest =
+    new ProductionVinylDNSAuthenticator(mockAuthenticator, mockAuthPrincipalProvider)
 
   "VinylDNSAuthenticator" should {
     "use Crypto" in {
@@ -74,7 +75,7 @@ class VinylDNSAuthenticatorSpec
         .when(mockAuthenticator)
         .authenticateReq(any[HttpRequest], any[List[String]], any[String], any[String])
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Right(okUserAuth)
     }
     "fail if missing Authorization header" in {
@@ -95,7 +96,7 @@ class VinylDNSAuthenticatorSpec
         .when(mockAuthenticator)
         .authenticateReq(any[HttpRequest], any[List[String]], any[String], any[String])
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Left(AuthMissing("Authorization header not found"))
     }
     "fail if Authorization header can not be parsed" in {
@@ -110,7 +111,7 @@ class VinylDNSAuthenticatorSpec
       val context: RequestContext = mock[RequestContext]
       doReturn(httpRequest).when(context).request
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Left(AuthRejected("Authorization header could not be parsed"))
     }
     "fail if the access key is missing" in {
@@ -133,7 +134,7 @@ class VinylDNSAuthenticatorSpec
         .when(mockAuthenticator)
         .extractAccessKey(any[String])
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Left(AuthMissing("accessKey not found"))
     }
     "fail if the access key can not be retrieved" in {
@@ -156,7 +157,7 @@ class VinylDNSAuthenticatorSpec
         .when(mockAuthenticator)
         .extractAccessKey(any[String])
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Left(AuthRejected("Invalid authorization header"))
     }
     "fail if the user is locked" in {
@@ -182,7 +183,7 @@ class VinylDNSAuthenticatorSpec
         .when(mockAuthPrincipalProvider)
         .getAuthPrincipal(any[String])
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Left(AccountLocked("Account with username locked is locked"))
     }
     "fail if the user can not be found" in {
@@ -209,7 +210,7 @@ class VinylDNSAuthenticatorSpec
         .when(mockAuthPrincipalProvider)
         .getAuthPrincipal(any[String])
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Left(AuthRejected("Account with accessKey fakeKey specified was not found"))
     }
     "fail if signatures can not be validated" in {
@@ -240,7 +241,7 @@ class VinylDNSAuthenticatorSpec
         .when(mockAuthenticator)
         .authenticateReq(any[HttpRequest], any[List[String]], any[String], any[String])
 
-      val result = underTest.apply(context, "").unsafeRunSync()
+      val result = underTest.authenticate(context, "").unsafeRunSync()
       result shouldBe Left(AuthRejected("Request signature could not be validated"))
     }
   }

modules/api/src/test/scala/vinyldns/api/route/VinylDNSDirectivesSpec.scala

@@ -26,6 +26,7 @@ import org.mockito.Matchers._
 import org.mockito.Mockito._
 import org.scalatest.mockito.MockitoSugar
 import org.scalatest.{BeforeAndAfterEach, Matchers, OneInstancePerTest, WordSpec}
+import vinyldns.core.domain.auth.AuthPrincipal
 import vinyldns.core.route.Monitor
 
 import scala.util.Failure
@@ -43,6 +44,9 @@ class VinylDNSDirectivesSpec
   private val mockLatency = mock[Histogram]
   private val mockErrors = mock[Meter]
 
+  val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator(
+    mock[AuthPrincipal])
+
   class TestMonitor extends Monitor("test") {
     override val latency: Histogram = mockLatency
     override val errors: Meter = mockErrors

modules/api/src/test/scala/vinyldns/api/route/VinylDNSServiceSpec.scala

@@ -22,6 +22,7 @@ import akka.event.Logging._
 import akka.http.scaladsl.model.headers.RawHeader
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.server.directives.LogEntry
+import vinyldns.core.domain.auth.AuthPrincipal
 
 class VinylDNSServiceSpec
     extends WordSpec
@@ -30,6 +31,9 @@ class VinylDNSServiceSpec
     with OneInstancePerTest
     with VinylDNSDirectives {
 
+  val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator(
+    mock[AuthPrincipal])
+
   private def buildMockRequest(
       path: String = "/path/to/resource",
       body: String = "request body") = {

modules/api/src/test/scala/vinyldns/api/route/ZoneRoutingSpec.scala

@@ -16,12 +16,10 @@
 
 package vinyldns.api.route
 
-import akka.actor.ActorSystem
 import akka.http.scaladsl.model.StatusCodes._
 import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest}
-import akka.http.scaladsl.server.{Directives, RequestContext, Route}
+import akka.http.scaladsl.server.{Directives, Route}
 import akka.http.scaladsl.testkit.ScalatestRouteTest
-import cats.effect._
 import org.json4s.JsonDSL._
 import org.json4s._
 import org.json4s.jackson.JsonMethods._
@@ -46,8 +44,6 @@ class ZoneRoutingSpec
     with Matchers
     with GroupTestData {
 
-  def actorRefFactory: ActorSystem = system
-
   private val okAuth = okGroupAuth
   private val alreadyExists = Zone("already.exists.", "test@test.com")
   private val notFound = Zone("not.found.", "test@test.com")
@@ -329,10 +325,7 @@ class ZoneRoutingSpec
 
   val zoneService: ZoneServiceAlgebra = TestZoneService
 
-  override def vinyldnsAuthenticator(
-      ctx: RequestContext,
-      content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] =
-    IO.pure(Right(okAuth))
+  val vinylDNSAuthenticator = new TestVinylDNSAuthenticator(okAuth)
 
   def zoneJson(name: String, email: String): String =
     zoneJson(Zone(name, email, connection = null, created = null, status = null, id = null))

modules/api/src/universal/conf/application.conf

@@ -28,6 +28,8 @@ vinyldns {
     type = "vinyldns.core.crypto.NoOpCrypto"
   }
 
+  data-stores = ["mysql", "dynamodb"]
+  
   # default settings point to the setup from docker compose
   mysql {
     settings {
@@ -52,67 +54,50 @@ vinyldns {
     }
   }
 
-  # dynamodb settings, for local docker compose the secrets are not needed
-  dynamo {
-    key = "x"
-    secret = "x"
-    endpoint = "http://vinyldns-dynamodb:8000"
-  }
-
-  # dynamodb table settings follow
-  zoneChanges {
-    dynamo {
-      tableName = "zoneChange"
-      provisionedReads = 30
-      provisionedWrites = 30
+  dynamodb {
+    # dynamodb settings, for local docker compose the secrets are not needed
+    settings {
+      key = "x"
+      secret = "x"
+      endpoint = "http://vinyldns-dynamodb:8000"
     }
-  }
 
-  recordSet {
-    dynamo {
-      tableName = "recordSet"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
-
-  recordChange {
-    dynamo {
-      tableName = "recordChange"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
-
-  users {
-    dynamo {
-      tableName = "users"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
-
-  groups {
-    dynamo {
-      tableName = "groups"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
-
-  groupChanges {
-    dynamo {
-      tableName = "groupChanges"
-      provisionedReads = 30
-      provisionedWrites = 30
-    }
-  }
-
-  membership {
-    dynamo {
-      tableName = "membership"
-      provisionedReads = 30
-      provisionedWrites = 30
+    repositories {
+      record-set {
+        table-name = "recordSet"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
+      record-change {
+        table-name = "recordChange"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
+      zone-change {
+        table-name = "zoneChange"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
+      user {
+        table-name = "users"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
+      group {
+        table-name = "groups"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
+      group-change {
+        table-name = "groupChanges"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
+      membership {
+        table-name = "membership"
+        provisioned-reads = 30
+        provisioned-writes = 30
+      }
     }
   }
 

modules/core/src/main/scala/vinyldns/core/repository/DataStoreLoader.scala

@@ -20,27 +20,33 @@ import cats.data._
 import cats.effect.IO
 import cats.implicits._
 import vinyldns.core.crypto.CryptoAlgebra
+import org.slf4j.LoggerFactory
 import vinyldns.core.repository.RepositoryName._
 
 import scala.reflect.ClassTag
 
 object DataStoreLoader {
+
+  private val logger = LoggerFactory.getLogger("DataStoreLoader")
+
   def loadAll[A <: DataAccessor](
       configs: List[DataStoreConfig],
       crypto: CryptoAlgebra,
-      dataAccessorProvider: DataAccessorProvider[A]): IO[DataAccessor] =
+      dataAccessorProvider: DataAccessorProvider[A]): IO[A] =
     for {
       activeConfigs <- IO.fromEither(getValidatedConfigs(configs, dataAccessorProvider.repoNames))
       dataStores <- activeConfigs.map(load(_, crypto)).parSequence
       accessor <- IO.fromEither(generateAccessor(dataStores, dataAccessorProvider))
     } yield accessor
 
-  def load(config: DataStoreConfig, crypto: CryptoAlgebra): IO[(DataStoreConfig, DataStore)] =
+  def load(config: DataStoreConfig, crypto: CryptoAlgebra): IO[(DataStoreConfig, DataStore)] = {
+    logger.error(s"Attempting to load repos ${config.repositories.keys} from ${config.className}")
     for {
       className <- IO.pure(config.className)
       provider <- IO(Class.forName(className).newInstance.asInstanceOf[DataStoreProvider])
       dataStore <- provider.load(config, crypto)
     } yield (config, dataStore)
+  }
 
   /*
    * Validates that there's exactly one repo defined across all datastore configs. Returns only

modules/dynamodb/src/it/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProviderIntegrationSpec.scala

@@ -16,6 +16,7 @@
 
 package vinyldns.dynamodb.repository
 
+import cats.effect.IO
 import cats.implicits._
 import com.amazonaws.services.dynamodbv2.model.DeleteTableRequest
 import com.typesafe.config.{Config, ConfigFactory}
@@ -77,10 +78,10 @@ class DynamoDBDataStoreProviderIntegrationSpec extends DynamoDBIntegrationSpec {
       )
       val userRepo = dataStore.get[UserRepository](user)
 
-      val save = userRepo.map(_.save(testUser)).parSequence
+      val save = userRepo.map(_.save(testUser)).sequence[IO, User]
       save.unsafeRunSync() shouldBe Some(testUser)
 
-      val get = userRepo.map(_.getUser(testUser.id)).parSequence
+      val get = userRepo.map(_.getUser(testUser.id)).sequence[IO, Option[User]]
       get.unsafeRunSync().flatten shouldBe Some(testUser)
     }
   }

modules/dynamodb/src/main/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProvider.scala

@@ -74,10 +74,17 @@ class DynamoDBDataStoreProvider extends DataStoreProvider {
 
     def initializeSingleRepo[T <: Repository](
         repoName: RepositoryName,
-        fn: DynamoDBRepositorySettings => IO[T]): IO[Option[T]] = {
-      logger.info(s"Loading dynamodb repo for type: $repoName")
-      repoSettings.get(repoName).map(fn(_)).parSequence
-    }
+        fn: DynamoDBRepositorySettings => IO[T]): IO[Option[T]] =
+      repoSettings
+        .get(repoName)
+        .map { configuredOn =>
+          for {
+            _ <- IO(logger.error(s"Loading dynamodb repo for type: $repoName"))
+            repo <- fn(configuredOn)
+            _ <- IO(logger.error(s"Completed dynamodb load for type: $repoName"))
+          } yield repo
+        }
+        .sequence
 
     (
       initializeSingleRepo[UserRepository](

project/build.properties

@@ -1 +1 @@
-sbt.version=1.1.4
+sbt.version=1.1.6