From 675cd110d0acfb01a0996b9ffe9e642e604ba61f Mon Sep 17 00:00:00 2001 From: Rebecca Star Date: Thu, 20 Sep 2018 10:58:04 -0400 Subject: [PATCH] Boot load repos in api (#214) * conf changes to dynamically load * have boot dynamically load repos based on config * update sbt version --- .travis.yml | 3 + docker/api/docker.conf | 104 ++++++++---------- modules/api/src/it/resources/application.conf | 50 ++++++--- .../ZoneCommandHandlerIntegrationSpec.scala | 80 +++++++------- .../repository/mysql/TestMySqlInstance.scala | 8 +- .../api/src/main/resources/application.conf | 38 +++++++ modules/api/src/main/resources/reference.conf | 64 ++--------- .../src/main/scala/vinyldns/api/Boot.scala | 93 ++++------------ .../scala/vinyldns/api/VinylDNSConfig.scala | 24 ++-- .../domain/auth/AuthPrincipalProvider.scala | 19 ---- .../api/domain/batch/BatchChangeService.scala | 14 +++ .../domain/membership/MembershipService.scala | 12 ++ .../api/domain/record/RecordSetService.scala | 16 +++ .../api/domain/zone/ZoneService.scala | 20 ++++ .../api/engine/ZoneCommandHandler.scala | 39 +++---- .../api/route/VinylDNSAuthentication.scala | 55 ++++----- .../api/route/VinylDNSDirectives.scala | 14 +-- .../vinyldns/api/route/VinylDNSService.scala | 12 +- .../api/src/test/resources/application.conf | 84 +++++--------- .../vinyldns/api/VinylDNSConfigSpec.scala | 51 +++------ .../api/route/BatchChangeRoutingSpec.scala | 1 + .../api/route/HealthCheckRoutingSpec.scala | 2 - .../api/route/MembershipRoutingSpec.scala | 1 + .../api/route/RecordSetRoutingSpec.scala | 8 +- .../api/route/TestVinylDNSAuthenticator.scala | 29 +++++ .../api/route/VinylDNSAuthenticatorSpec.scala | 21 ++-- .../api/route/VinylDNSDirectivesSpec.scala | 4 + .../api/route/VinylDNSServiceSpec.scala | 4 + .../vinyldns/api/route/ZoneRoutingSpec.scala | 11 +- .../api/src/universal/conf/application.conf | 103 ++++++++--------- .../core/repository/DataStoreLoader.scala | 10 +- ...moDBDataStoreProviderIntegrationSpec.scala | 5 +- .../DynamoDBDataStoreProvider.scala | 15 ++- project/build.properties | 2 +- 34 files changed, 490 insertions(+), 526 deletions(-) create mode 100644 modules/api/src/test/scala/vinyldns/api/route/TestVinylDNSAuthenticator.scala diff --git a/.travis.yml b/.travis.yml index e78aeb15f..4d56bf431 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ cache: directories: - $HOME/.ivy2/cache - $HOME/.sbt + timeout: 900 install: - rvm use 2.2.8 --install --fuzzy @@ -26,6 +27,8 @@ install: before_cache: # Cleanup the cached directories to avoid unnecessary cache updates (https://www.scala-sbt.org/1.0/docs/Travis-CI-with-sbt.html) + - sudo chown -R travis:travis $HOME/.sbt + - sudo chown -R travis:travis $HOME/.ivy2/cache - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete - find $HOME/.sbt -name "*.lock" -print -delete diff --git a/docker/api/docker.conf b/docker/api/docker.conf index 422250f50..1e62faa94 100644 --- a/docker/api/docker.conf +++ b/docker/api/docker.conf @@ -43,7 +43,7 @@ vinyldns { type = "vinyldns.core.crypto.NoOpCrypto" } - data-stores = ["mysql"] + data-stores = ["mysql", "dynamodb"] mysql { settings { @@ -75,69 +75,59 @@ vinyldns { } } - # default settings point to the docker compose setup - dynamo { - key = "x" - key = ${?AWS_ACCESS_KEY} - secret = "x" - secret = ${?AWS_SECRET_ACCESS_KEY} - endpoint = "http://vinyldns-dynamodb:8000" - endpoint = ${?DYNAMODB_ENDPOINT} - } - - zoneChanges { - dynamo { - tableName = "zoneChange" - provisionedReads = 30 - provisionedWrites = 30 + dynamodb { + settings { + # default settings point to the docker compose setup + key = "x" + key = ${?AWS_ACCESS_KEY} + secret = "x" + secret = ${?AWS_SECRET_ACCESS_KEY} + endpoint = "http://vinyldns-dynamodb:8000" + endpoint = ${?DYNAMODB_ENDPOINT} } - } - recordSet { - dynamo { - tableName = "recordSet" - provisionedReads = 30 - provisionedWrites = 30 - } - } + repositories { + zone-change { + table-name = "zoneChange" + provisioned-reads = 30 + provisioned-writes = 30 + } - recordChange { - dynamo { - tableName = "recordChange" - provisionedReads = 30 - provisionedWrites = 30 - } - } + record-set { + table-name = "recordSet" + provisioned-reads = 30 + provisioned-writes = 30 + } - users { - dynamo { - tableName = "users" - provisionedReads = 30 - provisionedWrites = 30 - } - } + record-change { + table-name = "recordChange" + provisioned-reads = 30 + provisioned-writes = 30 + } - groups { - dynamo { - tableName = "groups" - provisionedReads = 30 - provisionedWrites = 30 - } - } + user { + table-name = "users" + provisioned-reads = 30 + provisioned-writes = 30 + } - groupChanges { - dynamo { - tableName = "groupChanges" - provisionedReads = 30 - provisionedWrites = 30 - } - } + group { + table-name = "groups" + provisioned-reads = 30 + provisioned-writes = 30 + } - membership { - dynamo { - tableName = "membership" - provisionedReads = 30 - provisionedWrites = 30 + group-change { + table-name = "groupChanges" + provisioned-reads = 30 + provisioned-writes = 30 + } + + membership { + table-name = "membership" + provisioned-reads = 30 + provisioned-writes = 30 + } } } diff --git a/modules/api/src/it/resources/application.conf b/modules/api/src/it/resources/application.conf index 14b82f306..472962f0f 100644 --- a/modules/api/src/it/resources/application.conf +++ b/modules/api/src/it/resources/application.conf @@ -31,24 +31,42 @@ vinyldns { } } - recordSet { - # use the dummy store, this should only be used local - dummy = true - dynamo { - tableName = "recordSetTest" - provisionedReads=30 - provisionedWrites=30 + dynamodb.repositories { + record-set { + table-name = "recordSetTest" + provisioned-reads = 30 + provisioned-writes = 20 } - } - recordChange { - # use the dummy store, this should only be used local - dummy = true - - dynamo { - tableName = "recordChangeTest" - provisionedReads = 30 - provisionedWrites = 30 + record-change { + table-name = "recordChangeTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + zone-change { + table-name = "zoneChangesTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + user { + table-name = "usersTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + group { + table-name = "groupsTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + group-change { + table-name = "groupChangesTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + membership { + table-name = "membershipTest" + provisioned-reads = 30 + provisioned-writes = 20 } } diff --git a/modules/api/src/it/scala/vinyldns/api/engine/ZoneCommandHandlerIntegrationSpec.scala b/modules/api/src/it/scala/vinyldns/api/engine/ZoneCommandHandlerIntegrationSpec.scala index 423d7cb93..675bd4922 100644 --- a/modules/api/src/it/scala/vinyldns/api/engine/ZoneCommandHandlerIntegrationSpec.scala +++ b/modules/api/src/it/scala/vinyldns/api/engine/ZoneCommandHandlerIntegrationSpec.scala @@ -23,13 +23,14 @@ import cats.implicits._ import fs2.{Scheduler, Stream} import org.joda.time.DateTime import org.scalatest.concurrent.Eventually +import org.scalatest.mockito.MockitoSugar import org.scalatest.time.{Millis, Seconds, Span} import vinyldns.api.{DynamoDBApiIntegrationSpec, VinylDNSTestData} import vinyldns.api.domain.record.RecordSetChangeGenerator -import vinyldns.core.domain.batch.BatchChangeRepository import vinyldns.core.domain.record._ import vinyldns.api.domain.zone._ import vinyldns.api.engine.sqs.SqsConnection +import vinyldns.api.repository.ApiDataAccessor import vinyldns.dynamodb.repository.{ DynamoDBRecordChangeRepository, DynamoDBRecordSetRepository, @@ -37,14 +38,21 @@ import vinyldns.dynamodb.repository.{ DynamoDBZoneChangeRepository } import vinyldns.api.repository.mysql.TestMySqlInstance +import vinyldns.core.domain.membership.{ + GroupChangeRepository, + GroupRepository, + MembershipRepository, + UserRepository +} import vinyldns.core.domain.zone._ import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.ExecutionContext class ZoneCommandHandlerIntegrationSpec extends DynamoDBApiIntegrationSpec with VinylDNSTestData + with MockitoSugar with Eventually { import vinyldns.api.engine.sqs.SqsConverters._ @@ -65,11 +73,7 @@ class ZoneCommandHandlerIntegrationSpec PatienceConfig(timeout = Span(5, Seconds), interval = Span(500, Millis)) private implicit val ec: ExecutionContext = scala.concurrent.ExecutionContext.global - private var recordChangeRepo: RecordChangeRepository = _ - private var recordSetRepo: RecordSetRepository = _ - private var zoneChangeRepo: ZoneChangeRepository = _ - private var zoneRepo: ZoneRepository = _ - private var batchChangeRepo: BatchChangeRepository = _ + private var repositories: ApiDataAccessor = _ private var sqsConn: SqsConnection = _ private var str: Stream[IO, Unit] = _ private val stopSignal = fs2.async.signalOf[IO, Boolean](false).unsafeRunSync() @@ -119,38 +123,36 @@ class ZoneCommandHandlerIntegrationSpec } def setup(): Unit = { - val repos = ( - DynamoDBRecordChangeRepository(recordChangeStoreConfig, dynamoIntegrationConfig), + val dynamoRepos = ( DynamoDBRecordSetRepository(recordSetStoreConfig, dynamoIntegrationConfig), + DynamoDBRecordChangeRepository(recordChangeStoreConfig, dynamoIntegrationConfig), DynamoDBZoneChangeRepository(zoneChangeStoreConfig, dynamoIntegrationConfig) ).parTupled.unsafeRunSync() - recordChangeRepo = repos._1 - recordSetRepo = repos._2 - zoneChangeRepo = repos._3 - zoneRepo = TestMySqlInstance.zoneRepository - batchChangeRepo = TestMySqlInstance.batchChangeRepository + repositories = ApiDataAccessor( + mock[UserRepository], + mock[GroupRepository], + mock[MembershipRepository], + mock[GroupChangeRepository], + dynamoRepos._1, + dynamoRepos._2, + dynamoRepos._3, + TestMySqlInstance.zoneRepository, + TestMySqlInstance.batchChangeRepository + ) + sqsConn = SqsConnection() //seed items database - waitForSuccess(zoneRepo.save(testZone)) - waitForSuccess(recordChangeRepo.save(inDbRecordChange)) - waitForSuccess(recordChangeRepo.save(inDbRecordChangeForSyncTest)) - waitForSuccess(recordSetRepo.apply(inDbRecordChange)) - waitForSuccess(recordSetRepo.apply(inDbRecordChangeForSyncTest)) - waitForSuccess(zoneChangeRepo.save(inDbZoneChange)) - // Run a noop query to make sure recordSetRepo is up - waitForSuccess(recordSetRepo.listRecordSets("1", None, None, None)) + ( + repositories.zoneRepository.save(testZone), + repositories.recordChangeRepository.save(inDbRecordChange), + repositories.recordChangeRepository.save(inDbRecordChangeForSyncTest), + repositories.recordSetRepository.apply(inDbRecordChange), + repositories.recordSetRepository.apply(inDbRecordChangeForSyncTest), + repositories.zoneChangeRepository.save(inDbZoneChange)).parTupled.unsafeRunSync() - str = ZoneCommandHandler.mainFlow( - zoneRepo, - zoneChangeRepo, - recordSetRepo, - recordChangeRepo, - batchChangeRepo, - sqsConn, - 100.millis, - stopSignal) + str = ZoneCommandHandler.mainFlow(repositories, sqsConn, 100.millis, stopSignal) str.compile.drain.unsafeRunAsync { _ => () } @@ -168,7 +170,7 @@ class ZoneCommandHandlerIntegrationSpec sendCommand(change, sqsConn).unsafeRunSync() eventually { - val getZone = zoneRepo.getZone(testZone.id).unsafeToFuture() + val getZone = repositories.zoneRepository.getZone(testZone.id).unsafeToFuture() whenReady(getZone) { zn => zn.get.email shouldBe "updated@test.com" } @@ -180,7 +182,9 @@ class ZoneCommandHandlerIntegrationSpec RecordSetChangeGenerator.forUpdate(inDbRecordSet, inDbRecordSet.copy(ttl = 1234), testZone) sendCommand(change, sqsConn).unsafeRunSync() eventually { - val getRs = recordSetRepo.getRecordSet(testZone.id, inDbRecordSet.id).unsafeToFuture() + val getRs = repositories.recordSetRepository + .getRecordSet(testZone.id, inDbRecordSet.id) + .unsafeToFuture() whenReady(getRs) { rs => rs.get.ttl shouldBe 1234 } @@ -192,8 +196,9 @@ class ZoneCommandHandlerIntegrationSpec sendCommand(change, sqsConn).unsafeRunSync() eventually { val validatingQueries = for { - rs <- recordSetRepo.getRecordSet(testZone.id, inDbRecordSetForSyncTest.id) - ch <- recordChangeRepo.listRecordSetChanges(testZone.id) + rs <- repositories.recordSetRepository + .getRecordSet(testZone.id, inDbRecordSetForSyncTest.id) + ch <- repositories.recordChangeRepository.listRecordSetChanges(testZone.id) } yield (rs, ch) whenReady(validatingQueries.unsafeToFuture()) { data => @@ -210,9 +215,4 @@ class ZoneCommandHandlerIntegrationSpec } } } - - private def waitForSuccess[T](f: => IO[T]): T = { - val waiting = f.unsafeToFuture().recover { case _ => Thread.sleep(2000); waitForSuccess(f) } - Await.result[T](waiting, 15.seconds) - } } diff --git a/modules/api/src/it/scala/vinyldns/api/repository/mysql/TestMySqlInstance.scala b/modules/api/src/it/scala/vinyldns/api/repository/mysql/TestMySqlInstance.scala index aed8345cc..2b605ab55 100644 --- a/modules/api/src/it/scala/vinyldns/api/repository/mysql/TestMySqlInstance.scala +++ b/modules/api/src/it/scala/vinyldns/api/repository/mysql/TestMySqlInstance.scala @@ -20,11 +20,15 @@ import vinyldns.api.VinylDNSConfig import vinyldns.api.crypto.Crypto import vinyldns.core.domain.batch.BatchChangeRepository import vinyldns.core.domain.zone.{ZoneChangeRepository, ZoneRepository} -import vinyldns.core.repository.{DataStore, RepositoryName} +import vinyldns.core.repository.{DataStore, DataStoreConfig, RepositoryName} object TestMySqlInstance { + + lazy val mySqlConfig: DataStoreConfig = + pureconfig.loadConfigOrThrow[DataStoreConfig](VinylDNSConfig.vinyldnsConfig, "mysql") + lazy val instance: DataStore = - new MySqlDataStoreProvider().load(VinylDNSConfig.mySqlConfig, Crypto.instance).unsafeRunSync() + new MySqlDataStoreProvider().load(mySqlConfig, Crypto.instance).unsafeRunSync() lazy val zoneRepository: ZoneRepository = instance.get[ZoneRepository](RepositoryName.zone).get diff --git a/modules/api/src/main/resources/application.conf b/modules/api/src/main/resources/application.conf index 2580d8988..3feaac590 100644 --- a/modules/api/src/main/resources/application.conf +++ b/modules/api/src/main/resources/application.conf @@ -46,6 +46,44 @@ vinyldns { } } + dynamodb.repositories { + record-set { + table-name = "recordSetTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + record-change { + table-name = "recordChangeTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + zone-change { + table-name = "zoneChangesTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + user { + table-name = "usersTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + group { + table-name = "groupsTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + group-change { + table-name = "groupChangesTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + membership { + table-name = "membershipTest" + provisioned-reads = 30 + provisioned-writes = 20 + } + } + sync-delay = 10000 # 10 second delay for resyncing zone batch-change-limit = 20 # Max change limit per batch request diff --git a/modules/api/src/main/resources/reference.conf b/modules/api/src/main/resources/reference.conf index d3b53c476..c5bc26a6d 100644 --- a/modules/api/src/main/resources/reference.conf +++ b/modules/api/src/main/resources/reference.conf @@ -33,7 +33,7 @@ vinyldns { port = 9000 } - data-stores = ["mysql"] + data-stores = ["mysql", "dynamodb"] mysql { class-name = "vinyldns.api.repository.mysql.MySqlDataStoreProvider" @@ -59,60 +59,18 @@ vinyldns { } } - dynamo { - key = "vinyldnsTest" - secret = "notNeededForDynamoDbLocal" - endpoint = "http://127.0.0.1:19000" - region = "us-east-1" # note: we are always in us-east-1, but this can be overridden - } + dynamodb { + class-name = "vinyldns.dynamodb.repository.DynamoDBDataStoreProvider" - zoneChanges { - dynamo { - tableName = "zoneChanges" - provisionedReads=30 - provisionedWrites=30 + settings { + key = "vinyldnsTest" + secret = "notNeededForDynamoDbLocal" + endpoint = "http://127.0.0.1:19000" + region = "us-east-1" # note: we are always in us-east-1, but this can be overridden } - } - recordSet { - dynamo { - tableName = "recordSet" - provisionedReads=30 - provisionedWrites=30 - } - } - recordChange { - dynamo { - tableName = "recordChange" - provisionedReads=30 - provisionedWrites=30 - } - } - users { - dynamo { - tableName = "users" - provisionedReads=30 - provisionedWrites=30 - } - } - groups { - dynamo { - tableName = "groups" - provisionedReads=30 - provisionedWrites=30 - } - } - groupChanges { - dynamo { - tableName = "groupChanges" - provisionedReads=30 - provisionedWrites=30 - } - } - membership { - dynamo { - tableName = "membership" - provisionedReads=30 - provisionedWrites=30 + + repositories { + # override } } diff --git a/modules/api/src/main/scala/vinyldns/api/Boot.scala b/modules/api/src/main/scala/vinyldns/api/Boot.scala index 18d4f3a36..51af54b4c 100644 --- a/modules/api/src/main/scala/vinyldns/api/Boot.scala +++ b/modules/api/src/main/scala/vinyldns/api/Boot.scala @@ -32,14 +32,10 @@ import vinyldns.api.domain.record.RecordSetService import vinyldns.api.domain.zone._ import vinyldns.api.engine.ProductionZoneCommandHandler import vinyldns.api.engine.sqs.{SqsCommandBus, SqsConnection} -import vinyldns.dynamodb.repository._ -import vinyldns.api.repository.TestDataLoader -import vinyldns.api.repository.mysql.MySqlDataStoreProvider +import vinyldns.api.repository.{ApiDataAccessor, ApiDataAccessorProvider, TestDataLoader} import vinyldns.api.route.{HealthService, VinylDNSService} import vinyldns.core.VinylDNSMetrics -import vinyldns.core.domain.batch.BatchChangeRepository -import vinyldns.core.domain.zone.ZoneRepository -import vinyldns.core.repository.{DataStoreStartupError, RepositoryName} +import vinyldns.core.repository.DataStoreLoader import scala.concurrent.{ExecutionContext, Future} import scala.io.{Codec, Source} @@ -65,41 +61,10 @@ object Boot extends App { for { banner <- vinyldnsBanner() crypto <- IO(Crypto.instance) // load crypto - // TODO datastore loading will not be hardcoded by type here - mySqlDataStore <- new MySqlDataStoreProvider().load(VinylDNSConfig.mySqlConfig, crypto) - zoneRepo <- IO.fromEither( - mySqlDataStore - .get[ZoneRepository](RepositoryName.zone) - .toRight[Throwable](DataStoreStartupError("Missing zone repository"))) - batchChangeRepo <- IO.fromEither( - mySqlDataStore - .get[BatchChangeRepository](RepositoryName.batchChange) - .toRight[Throwable](DataStoreStartupError("Missing zone repository"))) - // TODO this also will all be removed with dynamic loading - userRepo <- DynamoDBUserRepository( - VinylDNSConfig.usersStoreConfig, - VinylDNSConfig.dynamoConfig, - crypto - ) - groupRepo <- DynamoDBGroupRepository( - VinylDNSConfig.groupsStoreConfig, - VinylDNSConfig.dynamoConfig) - membershipRepo <- DynamoDBMembershipRepository( - VinylDNSConfig.membershipStoreConfig, - VinylDNSConfig.dynamoConfig) - groupChangeRepo <- DynamoDBGroupChangeRepository( - VinylDNSConfig.groupChangesStoreConfig, - VinylDNSConfig.dynamoConfig) - recordSetRepo <- DynamoDBRecordSetRepository( - VinylDNSConfig.recordSetStoreConfig, - VinylDNSConfig.dynamoConfig) - recordChangeRepo <- DynamoDBRecordChangeRepository( - VinylDNSConfig.recordChangeStoreConfig, - VinylDNSConfig.dynamoConfig) - zoneChangeRepo <- DynamoDBZoneChangeRepository( - VinylDNSConfig.zoneChangeStoreConfig, - VinylDNSConfig.dynamoConfig) - _ <- TestDataLoader.loadTestData(userRepo) + repoConfigs <- VinylDNSConfig.dataStoreConfigs + repositories <- DataStoreLoader + .loadAll[ApiDataAccessor](repoConfigs, crypto, ApiDataAccessorProvider) + _ <- TestDataLoader.loadTestData(repositories.userRepository) sqsConfig <- IO(VinylDNSConfig.sqsConfig) sqsConnection <- IO(SqsConnection(sqsConfig)) processingDisabled <- IO(VinylDNSConfig.vinyldnsConfig.getBoolean("processing-disabled")) @@ -109,47 +74,26 @@ object Boot extends App { batchChangeLimit <- IO(VinylDNSConfig.vinyldnsConfig.getInt("batch-change-limit")) syncDelay <- IO(VinylDNSConfig.vinyldnsConfig.getInt("sync-delay")) _ <- fs2.async.start( - ProductionZoneCommandHandler.run( - sqsConnection, - processingSignal, - zoneRepo, - zoneChangeRepo, - recordChangeRepo, - recordSetRepo, - batchChangeRepo, - sqsConfig)) + ProductionZoneCommandHandler.run(sqsConnection, processingSignal, repositories, sqsConfig)) } yield { val zoneValidations = new ZoneValidations(syncDelay) val batchChangeValidations = new BatchChangeValidations(batchChangeLimit, AccessValidations) val commandBus = new SqsCommandBus(sqsConnection) - val membershipService = - new MembershipService(groupRepo, userRepo, membershipRepo, zoneRepo, groupChangeRepo) + val membershipService = MembershipService(repositories) val connectionValidator = new ZoneConnectionValidator(VinylDNSConfig.defaultZoneConnection) - val recordSetService = new RecordSetService( - zoneRepo, - recordSetRepo, - recordChangeRepo, - userRepo, - commandBus, - AccessValidations) - val zoneService = new ZoneService( - zoneRepo, - groupRepo, - userRepo, - zoneChangeRepo, + val recordSetService = RecordSetService(repositories, commandBus, AccessValidations) + val zoneService = ZoneService( + repositories, connectionValidator, commandBus, zoneValidations, AccessValidations) - val healthService = new HealthService(zoneRepo) - val batchChangeConverter = new BatchChangeConverter(batchChangeRepo, commandBus) - val batchChangeService = new BatchChangeService( - zoneRepo, - recordSetRepo, - batchChangeValidations, - batchChangeRepo, - batchChangeConverter) + val healthService = new HealthService(repositories.zoneRepository) + val batchChangeConverter = + new BatchChangeConverter(repositories.batchChangeRepository, commandBus) + val batchChangeService = + BatchChangeService(repositories, batchChangeValidations, batchChangeConverter) val collectorRegistry = CollectorRegistry.defaultRegistry val vinyldnsService = new VinylDNSService( membershipService, @@ -158,7 +102,10 @@ object Boot extends App { healthService, recordSetService, batchChangeService, - collectorRegistry) + collectorRegistry, + repositories.userRepository, + repositories.membershipRepository + ) DefaultExports.initialize() collectorRegistry.register(new DropwizardExports(VinylDNSMetrics.metricsRegistry)) diff --git a/modules/api/src/main/scala/vinyldns/api/VinylDNSConfig.scala b/modules/api/src/main/scala/vinyldns/api/VinylDNSConfig.scala index ade5a2469..dcdc4f7ed 100644 --- a/modules/api/src/main/scala/vinyldns/api/VinylDNSConfig.scala +++ b/modules/api/src/main/scala/vinyldns/api/VinylDNSConfig.scala @@ -17,7 +17,10 @@ package vinyldns.api import akka.actor.ActorSystem +import cats.effect.IO +import cats.implicits._ import com.typesafe.config.{Config, ConfigFactory} +import pureconfig.module.catseffect.loadConfigF import pureconfig.{CamelCase, ConfigFieldMapping, ProductHint} import vinyldns.api.VinylDNSConfig.vinyldnsConfig import vinyldns.api.crypto.Crypto @@ -33,21 +36,18 @@ object VinylDNSConfig { lazy val config: Config = ConfigFactory.load() lazy val vinyldnsConfig: Config = config.getConfig("vinyldns") - lazy val dynamoConfig = DynamoConfig.dynamoConfig - lazy val zoneChangeStoreConfig: DynamoDBRepositorySettings = DynamoConfig.zoneChangeStoreConfig - lazy val recordSetStoreConfig: DynamoDBRepositorySettings = DynamoConfig.recordSetStoreConfig - lazy val recordChangeStoreConfig: DynamoDBRepositorySettings = - DynamoConfig.recordChangeStoreConfig - lazy val usersStoreConfig: DynamoDBRepositorySettings = DynamoConfig.usersStoreConfig - lazy val groupsStoreConfig: DynamoDBRepositorySettings = DynamoConfig.groupsStoreConfig - lazy val groupChangesStoreConfig: DynamoDBRepositorySettings = - DynamoConfig.groupChangesStoreConfig - lazy val membershipStoreConfig: DynamoDBRepositorySettings = DynamoConfig.membershipStoreConfig + lazy val dataStoreConfigs: IO[List[DataStoreConfig]] = + vinyldnsConfig + .getStringList("data-stores") + .asScala + .toList + .map { configKey => + loadConfigF[IO, DataStoreConfig](vinyldnsConfig, configKey) + } + .parSequence lazy val restConfig: Config = vinyldnsConfig.getConfig("rest") lazy val monitoringConfig: Config = vinyldnsConfig.getConfig("monitoring") - lazy val mySqlConfig: DataStoreConfig = - pureconfig.loadConfigOrThrow[DataStoreConfig](vinyldnsConfig, "mysql") lazy val sqsConfig: Config = vinyldnsConfig.getConfig("sqs") lazy val cryptoConfig: Config = vinyldnsConfig.getConfig("crypto") lazy val system: ActorSystem = ActorSystem("VinylDNS", VinylDNSConfig.config) diff --git a/modules/api/src/main/scala/vinyldns/api/domain/auth/AuthPrincipalProvider.scala b/modules/api/src/main/scala/vinyldns/api/domain/auth/AuthPrincipalProvider.scala index e0460624b..92108a992 100644 --- a/modules/api/src/main/scala/vinyldns/api/domain/auth/AuthPrincipalProvider.scala +++ b/modules/api/src/main/scala/vinyldns/api/domain/auth/AuthPrincipalProvider.scala @@ -17,10 +17,7 @@ package vinyldns.api.domain.auth import cats.effect._ -import vinyldns.api.VinylDNSConfig -import vinyldns.api.crypto.Crypto import vinyldns.core.domain.membership.{MembershipRepository, User, UserRepository} -import vinyldns.dynamodb.repository.{DynamoDBMembershipRepository, DynamoDBUserRepository} import vinyldns.core.domain.auth.AuthPrincipal import vinyldns.core.route.Monitored @@ -52,19 +49,3 @@ class MembershipAuthPrincipalProvider( membershipRepo.getGroupsForUser(userId) } } - -object MembershipAuthPrincipalProvider { - // TODO this has to be dynamic!!! - val userRepository: UserRepository = - DynamoDBUserRepository( - VinylDNSConfig.usersStoreConfig, - VinylDNSConfig.dynamoConfig, - Crypto.instance) - .unsafeRunSync() - val membershipRepository: MembershipRepository = - DynamoDBMembershipRepository(VinylDNSConfig.membershipStoreConfig, VinylDNSConfig.dynamoConfig) - .unsafeRunSync() - - def apply(): MembershipAuthPrincipalProvider = - new MembershipAuthPrincipalProvider(userRepository, membershipRepository) -} diff --git a/modules/api/src/main/scala/vinyldns/api/domain/batch/BatchChangeService.scala b/modules/api/src/main/scala/vinyldns/api/domain/batch/BatchChangeService.scala index cd4ad626f..c57f52c6f 100644 --- a/modules/api/src/main/scala/vinyldns/api/domain/batch/BatchChangeService.scala +++ b/modules/api/src/main/scala/vinyldns/api/domain/batch/BatchChangeService.scala @@ -29,8 +29,22 @@ import vinyldns.core.domain.record.RecordType._ import vinyldns.core.domain.record.{RecordSet, RecordSetRepository} import vinyldns.core.domain.zone.ZoneRepository import vinyldns.api.domain.{RecordAlreadyExists, ZoneDiscoveryError} +import vinyldns.api.repository.ApiDataAccessor import vinyldns.core.domain.batch.{BatchChange, BatchChangeRepository, BatchChangeSummaryList} +object BatchChangeService { + def apply( + dataAccessor: ApiDataAccessor, + batchChangeValidations: BatchChangeValidationsAlgebra, + batchChangeConverter: BatchChangeConverterAlgebra): BatchChangeService = + new BatchChangeService( + dataAccessor.zoneRepository, + dataAccessor.recordSetRepository, + batchChangeValidations, + dataAccessor.batchChangeRepository, + batchChangeConverter) +} + class BatchChangeService( zoneRepository: ZoneRepository, recordSetRepository: RecordSetRepository, diff --git a/modules/api/src/main/scala/vinyldns/api/domain/membership/MembershipService.scala b/modules/api/src/main/scala/vinyldns/api/domain/membership/MembershipService.scala index b0e33c7e9..9ac66b2cd 100644 --- a/modules/api/src/main/scala/vinyldns/api/domain/membership/MembershipService.scala +++ b/modules/api/src/main/scala/vinyldns/api/domain/membership/MembershipService.scala @@ -18,11 +18,23 @@ package vinyldns.api.domain.membership import cats.implicits._ import vinyldns.api.Interfaces._ +import vinyldns.api.repository.ApiDataAccessor import vinyldns.core.domain.auth.AuthPrincipal import vinyldns.core.domain.membership.LockStatus.LockStatus import vinyldns.core.domain.zone.ZoneRepository import vinyldns.core.domain.membership._ +object MembershipService { + def apply(dataAccessor: ApiDataAccessor): MembershipService = + new MembershipService( + dataAccessor.groupRepository, + dataAccessor.userRepository, + dataAccessor.membershipRepository, + dataAccessor.zoneRepository, + dataAccessor.groupChangeRepository + ) +} + class MembershipService( groupRepo: GroupRepository, userRepo: UserRepository, diff --git a/modules/api/src/main/scala/vinyldns/api/domain/record/RecordSetService.scala b/modules/api/src/main/scala/vinyldns/api/domain/record/RecordSetService.scala index 99407b5e9..44e77c9ef 100644 --- a/modules/api/src/main/scala/vinyldns/api/domain/record/RecordSetService.scala +++ b/modules/api/src/main/scala/vinyldns/api/domain/record/RecordSetService.scala @@ -22,10 +22,26 @@ import vinyldns.core.domain.auth.AuthPrincipal import vinyldns.api.domain.engine.EngineCommandBus import vinyldns.core.domain.membership.{User, UserRepository} import vinyldns.api.domain.zone._ +import vinyldns.api.repository.ApiDataAccessor import vinyldns.api.route.ListRecordSetsResponse import vinyldns.core.domain.record._ import vinyldns.core.domain.zone.{Zone, ZoneCommandResult, ZoneRepository} +object RecordSetService { + def apply( + dataAccessor: ApiDataAccessor, + commandBus: EngineCommandBus, + accessValidation: AccessValidationAlgebra): RecordSetService = + new RecordSetService( + dataAccessor.zoneRepository, + dataAccessor.recordSetRepository, + dataAccessor.recordChangeRepository, + dataAccessor.userRepository, + commandBus, + accessValidation + ) +} + class RecordSetService( zoneRepository: ZoneRepository, recordSetRepository: RecordSetRepository, diff --git a/modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneService.scala b/modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneService.scala index e393721a7..d440ce4d5 100644 --- a/modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneService.scala +++ b/modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneService.scala @@ -21,9 +21,29 @@ import vinyldns.api.Interfaces._ import vinyldns.api.domain.AccessValidationAlgebra import vinyldns.core.domain.auth.AuthPrincipal import vinyldns.api.domain.engine.EngineCommandBus +import vinyldns.api.repository.ApiDataAccessor import vinyldns.core.domain.membership.{Group, GroupRepository, User, UserRepository} import vinyldns.core.domain.zone._ +object ZoneService { + def apply( + dataAccessor: ApiDataAccessor, + connectionValidator: ZoneConnectionValidatorAlgebra, + commandBus: EngineCommandBus, + zoneValidations: ZoneValidations, + accessValidation: AccessValidationAlgebra): ZoneService = + new ZoneService( + dataAccessor.zoneRepository, + dataAccessor.groupRepository, + dataAccessor.userRepository, + dataAccessor.zoneChangeRepository, + connectionValidator, + commandBus, + zoneValidations, + accessValidation + ) +} + class ZoneService( zoneRepository: ZoneRepository, groupRepository: GroupRepository, diff --git a/modules/api/src/main/scala/vinyldns/api/engine/ZoneCommandHandler.scala b/modules/api/src/main/scala/vinyldns/api/engine/ZoneCommandHandler.scala index 30667302b..8ed0e2abd 100644 --- a/modules/api/src/main/scala/vinyldns/api/engine/ZoneCommandHandler.scala +++ b/modules/api/src/main/scala/vinyldns/api/engine/ZoneCommandHandler.scala @@ -26,10 +26,10 @@ import fs2.async.mutable.Signal import org.slf4j.LoggerFactory import vinyldns.api.VinylDNSConfig import vinyldns.api.domain.dns.DnsConnection -import vinyldns.core.domain.record.{RecordChangeRepository, RecordSetChange, RecordSetRepository} -import vinyldns.core.domain.zone.{ZoneChange, ZoneChangeRepository, ZoneChangeType, ZoneRepository} +import vinyldns.core.domain.record.RecordSetChange +import vinyldns.core.domain.zone.{ZoneChange, ZoneChangeType} import vinyldns.api.engine.sqs.SqsConnection -import vinyldns.core.domain.batch.BatchChangeRepository +import vinyldns.api.repository.ApiDataAccessor import scala.collection.JavaConverters._ import scala.concurrent.ExecutionContext.Implicits.global @@ -63,11 +63,7 @@ object ZoneCommandHandler { extends ChangeRequest def mainFlow( - zoneRepository: ZoneRepository, - zoneChangeRepository: ZoneChangeRepository, - recordSetRepository: RecordSetRepository, - recordChangeRepository: RecordChangeRepository, - batchChangeRepository: BatchChangeRepository, + dataAccessor: ApiDataAccessor, sqsConnection: SqsConnection, pollingInterval: FiniteDuration, pauseSignal: Signal[IO, Boolean])(implicit scheduler: Scheduler): Stream[IO, Unit] = { @@ -79,10 +75,15 @@ object ZoneCommandHandler { val increaseTimeoutForZoneSyncs = changeVisibilityTimeoutForZoneSyncs(sqsConnection) // Handlers for each type of change request - val zoneChangeHandler = ZoneChangeHandler(zoneRepository, zoneChangeRepository) + val zoneChangeHandler = + ZoneChangeHandler(dataAccessor.zoneRepository, dataAccessor.zoneChangeRepository) val recordChangeHandler = - RecordSetChangeHandler(recordSetRepository, recordChangeRepository, batchChangeRepository) - val zoneSyncHandler = ZoneSyncHandler(recordSetRepository, recordChangeRepository) + RecordSetChangeHandler( + dataAccessor.recordSetRepository, + dataAccessor.recordChangeRepository, + dataAccessor.batchChangeRepository) + val zoneSyncHandler = + ZoneSyncHandler(dataAccessor.recordSetRepository, dataAccessor.recordChangeRepository) val changeRequestProcessor = processChangeRequests(zoneChangeHandler, recordChangeHandler, zoneSyncHandler) @@ -237,11 +238,7 @@ object ProductionZoneCommandHandler { def run( sqsConnection: SqsConnection, processingSignal: Signal[IO, Boolean], - zoneRepository: ZoneRepository, - zoneChangeRepository: ZoneChangeRepository, - recordChangeRepository: RecordChangeRepository, - recordSetRepository: RecordSetRepository, - batchChangeRepository: BatchChangeRepository, + dataAccessor: ApiDataAccessor, config: Config): IO[Unit] = { implicit val scheduler: Scheduler = Scheduler.fromScheduledExecutorService(Executors.newScheduledThreadPool(2)) @@ -250,15 +247,7 @@ object ProductionZoneCommandHandler { pollingInterval <- IO.pure( config.getDuration("polling-interval", TimeUnit.MILLISECONDS).milliseconds) flow <- ZoneCommandHandler - .mainFlow( - zoneRepository, - zoneChangeRepository, - recordSetRepository, - recordChangeRepository, - batchChangeRepository, - sqsConnection, - pollingInterval, - processingSignal) + .mainFlow(dataAccessor, sqsConnection, pollingInterval, processingSignal) .compile .drain } yield flow diff --git a/modules/api/src/main/scala/vinyldns/api/route/VinylDNSAuthentication.scala b/modules/api/src/main/scala/vinyldns/api/route/VinylDNSAuthentication.scala index 3bdced861..dd39293ae 100644 --- a/modules/api/src/main/scala/vinyldns/api/route/VinylDNSAuthentication.scala +++ b/modules/api/src/main/scala/vinyldns/api/route/VinylDNSAuthentication.scala @@ -21,7 +21,7 @@ import akka.http.scaladsl.server.RequestContext import cats.effect._ import cats.syntax.all._ import vinyldns.api.crypto.Crypto -import vinyldns.api.domain.auth.{AuthPrincipalProvider, MembershipAuthPrincipalProvider} +import vinyldns.api.domain.auth.AuthPrincipalProvider import vinyldns.core.crypto.CryptoAlgebra import vinyldns.core.domain.auth.AuthPrincipal import vinyldns.core.domain.membership.LockStatus @@ -34,9 +34,27 @@ final case class AuthMissing(msg: String) extends VinylDNSAuthenticationError(ms final case class AuthRejected(reason: String) extends VinylDNSAuthenticationError(reason) final case class AccountLocked(reason: String) extends VinylDNSAuthenticationError(reason) -trait VinylDNSAuthentication extends Monitored { - val authenticator: Aws4Authenticator - val authPrincipalProvider: AuthPrincipalProvider +trait VinylDNSAuthenticator { + def authenticate( + ctx: RequestContext, + content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] +} + +class ProductionVinylDNSAuthenticator( + val authenticator: Aws4Authenticator, + val authPrincipalProvider: AuthPrincipalProvider) + extends VinylDNSAuthenticator + with Monitored { + + def authenticate( + ctx: RequestContext, + content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] = + // Need to refactor getAuthPrincipal to be an IO[Either[E, A]] instead of how it is implemented. + getAuthPrincipal(ctx, content).attempt.flatMap { + case Left(e: VinylDNSAuthenticationError) => IO.pure(Left(e)) + case Right(ok) => IO.pure(Right(ok)) + case Left(e) => IO.raiseError(e) + } /** * Gets the auth header from the request. If the auth header is not found then the @@ -111,7 +129,7 @@ trait VinylDNSAuthentication extends Monitored { * @param ctx The Http Request Context * @return A Future containing the AuthPrincipal for the request. */ - def authenticate(ctx: RequestContext, content: String): IO[AuthPrincipal] = + def getAuthPrincipal(ctx: RequestContext, content: String): IO[AuthPrincipal] = for { authHeader <- getAuthHeader(ctx) regexMatch <- parseAuthHeader(authHeader) @@ -138,30 +156,3 @@ trait VinylDNSAuthentication extends Monitored { IO.raiseError(AuthRejected(s"Account with accessKey $accessKey specified was not found")) } } - -class VinylDNSAuthenticator( - val authenticator: Aws4Authenticator, - val authPrincipalProvider: AuthPrincipalProvider) - extends VinylDNSAuthentication { - - def apply( - ctx: RequestContext, - content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] = - // Need to refactor authenticate to be an IO[Either[E, A]] instead of how it is implemented, for the time being... - authenticate(ctx, content).attempt.flatMap { - case Left(e: VinylDNSAuthenticationError) => IO.pure(Left(e)) - case Right(ok) => IO.pure(Right(ok)) - case Left(e) => IO.raiseError(e) - } -} - -object VinylDNSAuthenticator { - lazy val aws4Authenticator = new Aws4Authenticator - lazy val authPrincipalProvider = MembershipAuthPrincipalProvider() - lazy val authenticator = new VinylDNSAuthenticator(aws4Authenticator, authPrincipalProvider) - - def apply( - ctx: RequestContext, - content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] = - authenticator.apply(ctx, content) -} diff --git a/modules/api/src/main/scala/vinyldns/api/route/VinylDNSDirectives.scala b/modules/api/src/main/scala/vinyldns/api/route/VinylDNSDirectives.scala index 64c2a6dcc..5a3d1ddf7 100644 --- a/modules/api/src/main/scala/vinyldns/api/route/VinylDNSDirectives.scala +++ b/modules/api/src/main/scala/vinyldns/api/route/VinylDNSDirectives.scala @@ -22,7 +22,6 @@ import akka.http.scaladsl.server._ import akka.http.scaladsl.server.directives.BasicDirectives import cats.data.Validated.{Invalid, Valid} import cats.data.ValidatedNel -import cats.effect._ import org.json4s.JsonDSL._ import org.json4s.jackson.JsonMethods._ import vinyldns.core.domain.auth.AuthPrincipal @@ -34,21 +33,14 @@ import scala.util.control.NonFatal trait VinylDNSDirectives extends Directives { - /** - * Authenticator that takes a request context and yields an Authentication, which is an Either - * that holds a Left - Rejection, or Right - AuthPrincipal. - * @return an Authentication with the AuthPrincipal as looked up from the request, or a Left(Rejection) - */ - def vinyldnsAuthenticator( - ctx: RequestContext, - content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] = - VinylDNSAuthenticator(ctx, content) + val vinylDNSAuthenticator: VinylDNSAuthenticator def authenticate: Directive1[AuthPrincipal] = extractExecutionContext.flatMap { implicit ec ⇒ extractRequestContext.flatMap { ctx => extractStrictEntity(10.seconds).flatMap { strictEntity => - onSuccess(vinyldnsAuthenticator(ctx, strictEntity.data.utf8String).unsafeToFuture()) + onSuccess( + vinylDNSAuthenticator.authenticate(ctx, strictEntity.data.utf8String).unsafeToFuture()) .flatMap { case Right(authPrincipal) ⇒ provide(authPrincipal) diff --git a/modules/api/src/main/scala/vinyldns/api/route/VinylDNSService.scala b/modules/api/src/main/scala/vinyldns/api/route/VinylDNSService.scala index b3cf04c7d..38c678c6e 100644 --- a/modules/api/src/main/scala/vinyldns/api/route/VinylDNSService.scala +++ b/modules/api/src/main/scala/vinyldns/api/route/VinylDNSService.scala @@ -25,10 +25,12 @@ import akka.http.scaladsl.server.directives.LogEntry import cats.effect.IO import fs2.async.mutable.Signal import io.prometheus.client.CollectorRegistry +import vinyldns.api.domain.auth.MembershipAuthPrincipalProvider import vinyldns.api.domain.batch.BatchChangeServiceAlgebra import vinyldns.api.domain.membership.MembershipServiceAlgebra import vinyldns.api.domain.record.RecordSetServiceAlgebra import vinyldns.api.domain.zone.ZoneServiceAlgebra +import vinyldns.core.domain.membership.{MembershipRepository, UserRepository} import scala.util.matching.Regex @@ -103,7 +105,9 @@ class VinylDNSService( val healthService: HealthService, val recordSetService: RecordSetServiceAlgebra, val batchChangeService: BatchChangeServiceAlgebra, - val collectorRegistry: CollectorRegistry) + val collectorRegistry: CollectorRegistry, + userRepository: UserRepository, + membershipRepository: MembershipRepository) extends VinylDNSDirectives with PingRoute with ZoneRoute @@ -117,6 +121,12 @@ class VinylDNSService( with VinylDNSJsonProtocol with JsonValidationRejection { + val aws4Authenticator = new Aws4Authenticator + val authPrincipalProvider = + new MembershipAuthPrincipalProvider(userRepository, membershipRepository) + val vinylDNSAuthenticator: VinylDNSAuthenticator = + new ProductionVinylDNSAuthenticator(aws4Authenticator, authPrincipalProvider) + // Authenticated routes must go first def authenticatedRoutes: server.Route = handleRejections(validationRejectionHandler)(authenticate { authPrincipal => diff --git a/modules/api/src/test/resources/application.conf b/modules/api/src/test/resources/application.conf index 316b26f60..0612b810c 100644 --- a/modules/api/src/test/resources/application.conf +++ b/modules/api/src/test/resources/application.conf @@ -30,67 +30,41 @@ vinyldns { } } - accounts { - dummy = true - - dynamo { - key = "dynamoKey" - secret = "dynamoSecret" - endpoint = "dynamoEndpoint" + dynamodb.repositories { + record-set { + table-name = "recordSetTest" + provisioned-reads = 30 + provisioned-writes = 20 } - } - - dynamo { - key="dynamoKey" - secret="dynamoSecret" - endpoint="dynamoEndpoint" - } - - zoneChanges { - dynamo { - tableName = "zoneChanges" - provisionedReads=40 - provisionedWrites=30 + record-change { + table-name = "recordChangeTest" + provisioned-reads = 30 + provisioned-writes = 20 } - } - - recordSet { - dynamo { - tableName = "recordSet" - provisionedReads=40 - provisionedWrites=30 + zone-change { + table-name = "zoneChangesTest" + provisioned-reads = 30 + provisioned-writes = 20 } - } - - recordChange { - dynamo { - tableName = "recordChange" - provisionedReads=40 - provisionedWrites=30 + user { + table-name = "usersTest" + provisioned-reads = 30 + provisioned-writes = 20 } - } - - groups { - dynamo { - tableName = "groups" - provisionedReads=40 - provisionedWrites=30 + group { + table-name = "groupsTest" + provisioned-reads = 30 + provisioned-writes = 20 } - } - - groupChanges { - dynamo { - tableName = "groupChanges" - provisionedReads=40 - provisionedWrites=30 + group-change { + table-name = "groupChangesTest" + provisioned-reads = 30 + provisioned-writes = 20 } - } - - membership { - dynamo { - tableName = "membership" - provisionedReads=40 - provisionedWrites=30 + membership { + table-name = "membershipTest" + provisioned-reads = 30 + provisioned-writes = 20 } } diff --git a/modules/api/src/test/scala/vinyldns/api/VinylDNSConfigSpec.scala b/modules/api/src/test/scala/vinyldns/api/VinylDNSConfigSpec.scala index 8cbd153ca..9f96fa16e 100644 --- a/modules/api/src/test/scala/vinyldns/api/VinylDNSConfigSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/VinylDNSConfigSpec.scala @@ -17,6 +17,7 @@ package vinyldns.api import org.scalatest.{Matchers, WordSpec} +import vinyldns.core.repository.RepositoryName._ class VinylDNSConfigSpec extends WordSpec with Matchers { @@ -26,46 +27,26 @@ class VinylDNSConfigSpec extends WordSpec with Matchers { restConfig.getInt("port") shouldBe 9000 } - "load the dynamo config" in { - val dynamoConfig = VinylDNSConfig.dynamoConfig - dynamoConfig.key shouldBe "dynamoKey" - dynamoConfig.secret shouldBe "dynamoSecret" - dynamoConfig.endpoint shouldBe "dynamoEndpoint" - } + "properly load the datastore configs" in { - "load the zone change repository config" in { - val config = VinylDNSConfig.zoneChangeStoreConfig - config.tableName shouldBe "zoneChanges" - config.provisionedReads shouldBe 40 - config.provisionedWrites shouldBe 30 + VinylDNSConfig.dataStoreConfigs.unsafeRunSync.length shouldBe 2 } + "assign the correct mysql repositories" in { + val mysqlConfig = + VinylDNSConfig.dataStoreConfigs.unsafeRunSync + .find(_.className == "vinyldns.api.repository.mysql.MySqlDataStoreProvider") + .get - "load the record change repository config" in { - val config = VinylDNSConfig.recordChangeStoreConfig - config.tableName shouldBe "recordChange" - config.provisionedReads shouldBe 40 - config.provisionedWrites shouldBe 30 + mysqlConfig.repositories.keys should contain theSameElementsAs Set(zone, batchChange) } + "assign the correct dynamodb repositories" in { + val dynamodbConfig = + VinylDNSConfig.dataStoreConfigs.unsafeRunSync + .find(_.className == "vinyldns.dynamodb.repository.DynamoDBDataStoreProvider") + .get - "load the membership repository config" in { - val config = VinylDNSConfig.membershipStoreConfig - config.tableName shouldBe "membership" - config.provisionedReads shouldBe 40 - config.provisionedWrites shouldBe 30 - } - - "load the record set repository config" in { - val config = VinylDNSConfig.recordSetStoreConfig - config.tableName shouldBe "recordSet" - config.provisionedReads shouldBe 40 - config.provisionedWrites shouldBe 30 - } - - "load the group repository config" in { - val config = VinylDNSConfig.groupsStoreConfig - config.tableName shouldBe "groups" - config.provisionedReads shouldBe 40 - config.provisionedWrites shouldBe 30 + dynamodbConfig.repositories.keys should contain theSameElementsAs + Set(user, group, membership, groupChange, recordSet, recordChange, zoneChange) } } } diff --git a/modules/api/src/test/scala/vinyldns/api/route/BatchChangeRoutingSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/BatchChangeRoutingSpec.scala index 3dc39a30d..11e057a10 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/BatchChangeRoutingSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/BatchChangeRoutingSpec.scala @@ -47,6 +47,7 @@ class BatchChangeRoutingSpec with GroupTestData { val batchChangeService: BatchChangeServiceAlgebra = TestBatchChangeService + val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator(okUserAuth) import vinyldns.core.domain.batch.SingleChangeStatus._ diff --git a/modules/api/src/test/scala/vinyldns/api/route/HealthCheckRoutingSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/HealthCheckRoutingSpec.scala index 6ba58a74b..063b46c09 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/HealthCheckRoutingSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/HealthCheckRoutingSpec.scala @@ -25,7 +25,6 @@ import org.mockito.Mockito.doReturn import org.scalatest.mockito.MockitoSugar import org.scalatest.{Matchers, OneInstancePerTest, WordSpec} import vinyldns.core.domain.zone.ZoneRepository - import cats.effect._ class HealthCheckRoutingSpec @@ -34,7 +33,6 @@ class HealthCheckRoutingSpec with Directives with HealthCheckRoute with VinylDNSJsonProtocol - with VinylDNSDirectives with OneInstancePerTest with Matchers with MockitoSugar { diff --git a/modules/api/src/test/scala/vinyldns/api/route/MembershipRoutingSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/MembershipRoutingSpec.scala index 7ccc6948b..730faf0da 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/MembershipRoutingSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/MembershipRoutingSpec.scala @@ -51,6 +51,7 @@ class MembershipRoutingSpec with BeforeAndAfterEach { val membershipService: MembershipService = mock[MembershipService] + val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator(okAuth) override protected def beforeEach(): Unit = reset(membershipService) diff --git a/modules/api/src/test/scala/vinyldns/api/route/RecordSetRoutingSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/RecordSetRoutingSpec.scala index df69db889..d28b564e1 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/RecordSetRoutingSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/RecordSetRoutingSpec.scala @@ -17,9 +17,8 @@ package vinyldns.api.route import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest, StatusCodes} -import akka.http.scaladsl.server.{Directives, RequestContext, Route} +import akka.http.scaladsl.server.{Directives, Route} import akka.http.scaladsl.testkit.ScalatestRouteTest -import cats.effect._ import org.joda.time.DateTime import org.json4s.JsonDSL._ import org.json4s._ @@ -482,10 +481,7 @@ class RecordSetRoutingSpec val recordSetService: RecordSetServiceAlgebra = new TestService - override def vinyldnsAuthenticator( - ctx: RequestContext, - content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] = - IO.pure(Right(okAuth)) + val vinylDNSAuthenticator = new TestVinylDNSAuthenticator(okAuth) private def rsJson(recordSet: RecordSet): String = compact(render(Extraction.decompose(recordSet))) diff --git a/modules/api/src/test/scala/vinyldns/api/route/TestVinylDNSAuthenticator.scala b/modules/api/src/test/scala/vinyldns/api/route/TestVinylDNSAuthenticator.scala new file mode 100644 index 000000000..05910ea53 --- /dev/null +++ b/modules/api/src/test/scala/vinyldns/api/route/TestVinylDNSAuthenticator.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2018 Comcast Cable Communications Management, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vinyldns.api.route + +import akka.http.scaladsl.server.RequestContext +import cats.effect.IO +import vinyldns.core.domain.auth.AuthPrincipal + +class TestVinylDNSAuthenticator(authPrincipal: AuthPrincipal) extends VinylDNSAuthenticator { + + def authenticate( + ctx: RequestContext, + content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] = + IO.pure(Right(authPrincipal)) +} diff --git a/modules/api/src/test/scala/vinyldns/api/route/VinylDNSAuthenticatorSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/VinylDNSAuthenticatorSpec.scala index af5843976..8fea2f78a 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/VinylDNSAuthenticatorSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/VinylDNSAuthenticatorSpec.scala @@ -24,7 +24,7 @@ import org.mockito.Mockito._ import org.scalatest.mockito.MockitoSugar import org.scalatest.{Matchers, WordSpec} import vinyldns.api.domain.auth.AuthPrincipalProvider -import vinyldns.api.{GroupTestData} +import vinyldns.api.GroupTestData import vinyldns.core.crypto.CryptoAlgebra class VinylDNSAuthenticatorSpec @@ -35,7 +35,8 @@ class VinylDNSAuthenticatorSpec private val mockAuthenticator = mock[Aws4Authenticator] private val mockAuthPrincipalProvider = mock[AuthPrincipalProvider] - private val underTest = new VinylDNSAuthenticator(mockAuthenticator, mockAuthPrincipalProvider) + private val underTest = + new ProductionVinylDNSAuthenticator(mockAuthenticator, mockAuthPrincipalProvider) "VinylDNSAuthenticator" should { "use Crypto" in { @@ -74,7 +75,7 @@ class VinylDNSAuthenticatorSpec .when(mockAuthenticator) .authenticateReq(any[HttpRequest], any[List[String]], any[String], any[String]) - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Right(okUserAuth) } "fail if missing Authorization header" in { @@ -95,7 +96,7 @@ class VinylDNSAuthenticatorSpec .when(mockAuthenticator) .authenticateReq(any[HttpRequest], any[List[String]], any[String], any[String]) - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Left(AuthMissing("Authorization header not found")) } "fail if Authorization header can not be parsed" in { @@ -110,7 +111,7 @@ class VinylDNSAuthenticatorSpec val context: RequestContext = mock[RequestContext] doReturn(httpRequest).when(context).request - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Left(AuthRejected("Authorization header could not be parsed")) } "fail if the access key is missing" in { @@ -133,7 +134,7 @@ class VinylDNSAuthenticatorSpec .when(mockAuthenticator) .extractAccessKey(any[String]) - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Left(AuthMissing("accessKey not found")) } "fail if the access key can not be retrieved" in { @@ -156,7 +157,7 @@ class VinylDNSAuthenticatorSpec .when(mockAuthenticator) .extractAccessKey(any[String]) - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Left(AuthRejected("Invalid authorization header")) } "fail if the user is locked" in { @@ -182,7 +183,7 @@ class VinylDNSAuthenticatorSpec .when(mockAuthPrincipalProvider) .getAuthPrincipal(any[String]) - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Left(AccountLocked("Account with username locked is locked")) } "fail if the user can not be found" in { @@ -209,7 +210,7 @@ class VinylDNSAuthenticatorSpec .when(mockAuthPrincipalProvider) .getAuthPrincipal(any[String]) - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Left(AuthRejected("Account with accessKey fakeKey specified was not found")) } "fail if signatures can not be validated" in { @@ -240,7 +241,7 @@ class VinylDNSAuthenticatorSpec .when(mockAuthenticator) .authenticateReq(any[HttpRequest], any[List[String]], any[String], any[String]) - val result = underTest.apply(context, "").unsafeRunSync() + val result = underTest.authenticate(context, "").unsafeRunSync() result shouldBe Left(AuthRejected("Request signature could not be validated")) } } diff --git a/modules/api/src/test/scala/vinyldns/api/route/VinylDNSDirectivesSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/VinylDNSDirectivesSpec.scala index 682490801..555b48581 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/VinylDNSDirectivesSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/VinylDNSDirectivesSpec.scala @@ -26,6 +26,7 @@ import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.mockito.MockitoSugar import org.scalatest.{BeforeAndAfterEach, Matchers, OneInstancePerTest, WordSpec} +import vinyldns.core.domain.auth.AuthPrincipal import vinyldns.core.route.Monitor import scala.util.Failure @@ -43,6 +44,9 @@ class VinylDNSDirectivesSpec private val mockLatency = mock[Histogram] private val mockErrors = mock[Meter] + val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator( + mock[AuthPrincipal]) + class TestMonitor extends Monitor("test") { override val latency: Histogram = mockLatency override val errors: Meter = mockErrors diff --git a/modules/api/src/test/scala/vinyldns/api/route/VinylDNSServiceSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/VinylDNSServiceSpec.scala index 124076c0b..470555cc9 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/VinylDNSServiceSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/VinylDNSServiceSpec.scala @@ -22,6 +22,7 @@ import akka.event.Logging._ import akka.http.scaladsl.model.headers.RawHeader import akka.http.scaladsl.model._ import akka.http.scaladsl.server.directives.LogEntry +import vinyldns.core.domain.auth.AuthPrincipal class VinylDNSServiceSpec extends WordSpec @@ -30,6 +31,9 @@ class VinylDNSServiceSpec with OneInstancePerTest with VinylDNSDirectives { + val vinylDNSAuthenticator: VinylDNSAuthenticator = new TestVinylDNSAuthenticator( + mock[AuthPrincipal]) + private def buildMockRequest( path: String = "/path/to/resource", body: String = "request body") = { diff --git a/modules/api/src/test/scala/vinyldns/api/route/ZoneRoutingSpec.scala b/modules/api/src/test/scala/vinyldns/api/route/ZoneRoutingSpec.scala index db0002f53..44efe1f62 100644 --- a/modules/api/src/test/scala/vinyldns/api/route/ZoneRoutingSpec.scala +++ b/modules/api/src/test/scala/vinyldns/api/route/ZoneRoutingSpec.scala @@ -16,12 +16,10 @@ package vinyldns.api.route -import akka.actor.ActorSystem import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpRequest} -import akka.http.scaladsl.server.{Directives, RequestContext, Route} +import akka.http.scaladsl.server.{Directives, Route} import akka.http.scaladsl.testkit.ScalatestRouteTest -import cats.effect._ import org.json4s.JsonDSL._ import org.json4s._ import org.json4s.jackson.JsonMethods._ @@ -46,8 +44,6 @@ class ZoneRoutingSpec with Matchers with GroupTestData { - def actorRefFactory: ActorSystem = system - private val okAuth = okGroupAuth private val alreadyExists = Zone("already.exists.", "test@test.com") private val notFound = Zone("not.found.", "test@test.com") @@ -329,10 +325,7 @@ class ZoneRoutingSpec val zoneService: ZoneServiceAlgebra = TestZoneService - override def vinyldnsAuthenticator( - ctx: RequestContext, - content: String): IO[Either[VinylDNSAuthenticationError, AuthPrincipal]] = - IO.pure(Right(okAuth)) + val vinylDNSAuthenticator = new TestVinylDNSAuthenticator(okAuth) def zoneJson(name: String, email: String): String = zoneJson(Zone(name, email, connection = null, created = null, status = null, id = null)) diff --git a/modules/api/src/universal/conf/application.conf b/modules/api/src/universal/conf/application.conf index 136da264a..217e2b124 100644 --- a/modules/api/src/universal/conf/application.conf +++ b/modules/api/src/universal/conf/application.conf @@ -28,6 +28,8 @@ vinyldns { type = "vinyldns.core.crypto.NoOpCrypto" } + data-stores = ["mysql", "dynamodb"] + # default settings point to the setup from docker compose mysql { settings { @@ -52,67 +54,50 @@ vinyldns { } } - # dynamodb settings, for local docker compose the secrets are not needed - dynamo { - key = "x" - secret = "x" - endpoint = "http://vinyldns-dynamodb:8000" - } - - # dynamodb table settings follow - zoneChanges { - dynamo { - tableName = "zoneChange" - provisionedReads = 30 - provisionedWrites = 30 + dynamodb { + # dynamodb settings, for local docker compose the secrets are not needed + settings { + key = "x" + secret = "x" + endpoint = "http://vinyldns-dynamodb:8000" } - } - recordSet { - dynamo { - tableName = "recordSet" - provisionedReads = 30 - provisionedWrites = 30 - } - } - - recordChange { - dynamo { - tableName = "recordChange" - provisionedReads = 30 - provisionedWrites = 30 - } - } - - users { - dynamo { - tableName = "users" - provisionedReads = 30 - provisionedWrites = 30 - } - } - - groups { - dynamo { - tableName = "groups" - provisionedReads = 30 - provisionedWrites = 30 - } - } - - groupChanges { - dynamo { - tableName = "groupChanges" - provisionedReads = 30 - provisionedWrites = 30 - } - } - - membership { - dynamo { - tableName = "membership" - provisionedReads = 30 - provisionedWrites = 30 + repositories { + record-set { + table-name = "recordSet" + provisioned-reads = 30 + provisioned-writes = 30 + } + record-change { + table-name = "recordChange" + provisioned-reads = 30 + provisioned-writes = 30 + } + zone-change { + table-name = "zoneChange" + provisioned-reads = 30 + provisioned-writes = 30 + } + user { + table-name = "users" + provisioned-reads = 30 + provisioned-writes = 30 + } + group { + table-name = "groups" + provisioned-reads = 30 + provisioned-writes = 30 + } + group-change { + table-name = "groupChanges" + provisioned-reads = 30 + provisioned-writes = 30 + } + membership { + table-name = "membership" + provisioned-reads = 30 + provisioned-writes = 30 + } } } diff --git a/modules/core/src/main/scala/vinyldns/core/repository/DataStoreLoader.scala b/modules/core/src/main/scala/vinyldns/core/repository/DataStoreLoader.scala index e26bc4c45..48d10d912 100644 --- a/modules/core/src/main/scala/vinyldns/core/repository/DataStoreLoader.scala +++ b/modules/core/src/main/scala/vinyldns/core/repository/DataStoreLoader.scala @@ -20,27 +20,33 @@ import cats.data._ import cats.effect.IO import cats.implicits._ import vinyldns.core.crypto.CryptoAlgebra +import org.slf4j.LoggerFactory import vinyldns.core.repository.RepositoryName._ import scala.reflect.ClassTag object DataStoreLoader { + + private val logger = LoggerFactory.getLogger("DataStoreLoader") + def loadAll[A <: DataAccessor]( configs: List[DataStoreConfig], crypto: CryptoAlgebra, - dataAccessorProvider: DataAccessorProvider[A]): IO[DataAccessor] = + dataAccessorProvider: DataAccessorProvider[A]): IO[A] = for { activeConfigs <- IO.fromEither(getValidatedConfigs(configs, dataAccessorProvider.repoNames)) dataStores <- activeConfigs.map(load(_, crypto)).parSequence accessor <- IO.fromEither(generateAccessor(dataStores, dataAccessorProvider)) } yield accessor - def load(config: DataStoreConfig, crypto: CryptoAlgebra): IO[(DataStoreConfig, DataStore)] = + def load(config: DataStoreConfig, crypto: CryptoAlgebra): IO[(DataStoreConfig, DataStore)] = { + logger.error(s"Attempting to load repos ${config.repositories.keys} from ${config.className}") for { className <- IO.pure(config.className) provider <- IO(Class.forName(className).newInstance.asInstanceOf[DataStoreProvider]) dataStore <- provider.load(config, crypto) } yield (config, dataStore) + } /* * Validates that there's exactly one repo defined across all datastore configs. Returns only diff --git a/modules/dynamodb/src/it/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProviderIntegrationSpec.scala b/modules/dynamodb/src/it/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProviderIntegrationSpec.scala index ae8d9b64f..3bc8e806b 100644 --- a/modules/dynamodb/src/it/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProviderIntegrationSpec.scala +++ b/modules/dynamodb/src/it/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProviderIntegrationSpec.scala @@ -16,6 +16,7 @@ package vinyldns.dynamodb.repository +import cats.effect.IO import cats.implicits._ import com.amazonaws.services.dynamodbv2.model.DeleteTableRequest import com.typesafe.config.{Config, ConfigFactory} @@ -77,10 +78,10 @@ class DynamoDBDataStoreProviderIntegrationSpec extends DynamoDBIntegrationSpec { ) val userRepo = dataStore.get[UserRepository](user) - val save = userRepo.map(_.save(testUser)).parSequence + val save = userRepo.map(_.save(testUser)).sequence[IO, User] save.unsafeRunSync() shouldBe Some(testUser) - val get = userRepo.map(_.getUser(testUser.id)).parSequence + val get = userRepo.map(_.getUser(testUser.id)).sequence[IO, Option[User]] get.unsafeRunSync().flatten shouldBe Some(testUser) } } diff --git a/modules/dynamodb/src/main/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProvider.scala b/modules/dynamodb/src/main/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProvider.scala index 504c631d3..93dc62ce5 100644 --- a/modules/dynamodb/src/main/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProvider.scala +++ b/modules/dynamodb/src/main/scala/vinyldns/dynamodb/repository/DynamoDBDataStoreProvider.scala @@ -74,10 +74,17 @@ class DynamoDBDataStoreProvider extends DataStoreProvider { def initializeSingleRepo[T <: Repository]( repoName: RepositoryName, - fn: DynamoDBRepositorySettings => IO[T]): IO[Option[T]] = { - logger.info(s"Loading dynamodb repo for type: $repoName") - repoSettings.get(repoName).map(fn(_)).parSequence - } + fn: DynamoDBRepositorySettings => IO[T]): IO[Option[T]] = + repoSettings + .get(repoName) + .map { configuredOn => + for { + _ <- IO(logger.error(s"Loading dynamodb repo for type: $repoName")) + repo <- fn(configuredOn) + _ <- IO(logger.error(s"Completed dynamodb load for type: $repoName")) + } yield repo + } + .sequence ( initializeSingleRepo[UserRepository]( diff --git a/project/build.properties b/project/build.properties index 64cf32f7f..d6e35076c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.1.4 +sbt.version=1.1.6