Merge pull request #1052 from Jay07GIT/hardcoded-limits

modules/api/src/main/resources/reference.conf

@@ -105,7 +105,18 @@ vinyldns {
     host = "127.0.0.1"
     port = 9000
   }
-
+  # limits for batchchange routing, membership routing , recordset routing , zone routing
+  api {
+    limits {
+      batchchange-routing-max-items-limit = 100
+      membership-routing-default-max-items = 100
+      membership-routing-max-items-limit = 1000
+      membership-routing-max-groups-list-limit = 1500
+      recordset-routing-default-max-items= 100
+      zone-routing-default-max-items = 100
+      zone-routing-max-items-limit = 100
+    }
+  }
   mysql {
     class-name = "vinyldns.mysql.repository.MySqlDataStoreProvider"
 

modules/api/src/main/scala/vinyldns/api/Boot.scala

@@ -27,7 +27,7 @@ import io.prometheus.client.dropwizard.DropwizardExports
 import io.prometheus.client.hotspot.DefaultExports
 import org.slf4j.LoggerFactory
 import vinyldns.api.backend.CommandHandler
-import vinyldns.api.config.VinylDNSConfig
+import vinyldns.api.config.{LimitsConfig, VinylDNSConfig}
 import vinyldns.api.domain.access.{AccessValidations, GlobalAcls}
 import vinyldns.api.domain.auth.MembershipAuthPrincipalProvider
 import vinyldns.api.domain.batch.{BatchChangeConverter, BatchChangeService, BatchChangeValidations}
@@ -124,6 +124,7 @@ object Boot extends App {
         vinyldnsConfig.scheduledChangesConfig
       )
       val membershipService = MembershipService(repositories)
+
       val connectionValidator =
         new ZoneConnectionValidator(
           backendResolver,
@@ -149,6 +150,16 @@ object Boot extends App {
         backendResolver,
         vinyldnsConfig.crypto
       )
+      //limits configured in reference.conf paasing here
+      val limits = LimitsConfig(
+        vinyldnsConfig.limitsconfig.BATCHCHANGE_ROUTING_MAX_ITEMS_LIMIT,
+        vinyldnsConfig.limitsconfig.MEMBERSHIP_ROUTING_DEFAULT_MAX_ITEMS,
+        vinyldnsConfig.limitsconfig.MEMBERSHIP_ROUTING_MAX_ITEMS_LIMIT,
+        vinyldnsConfig.limitsconfig.MEMBERSHIP_ROUTING_MAX_GROUPS_LIST_LIMIT,
+        vinyldnsConfig.limitsconfig.RECORDSET_ROUTING_DEFAULT_MAX_ITEMS,
+        vinyldnsConfig.limitsconfig.ZONE_ROUTING_DEFAULT_MAX_ITEMS,
+        vinyldnsConfig.limitsconfig.ZONE_ROUTING_MAX_ITEMS_LIMIT
+      )
       val healthService = new HealthService(
         messageQueue.healthCheck :: backendResolver.healthCheck(
           vinyldnsConfig.serverConfig.healthCheckTimeout
@@ -176,6 +187,7 @@ object Boot extends App {
       val collectorRegistry = CollectorRegistry.defaultRegistry
       val vinyldnsService = new VinylDNSService(
         membershipService,
+        limits,
         processingSignal,
         zoneService,
         healthService,

modules/api/src/main/scala/vinyldns/api/config/LimitsConfig.scala

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2018 Comcast Cable Communications Management, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package vinyldns.api.config
+
+import pureconfig.ConfigReader
+
+final case class LimitsConfig(
+    BATCHCHANGE_ROUTING_MAX_ITEMS_LIMIT: Int,
+    MEMBERSHIP_ROUTING_DEFAULT_MAX_ITEMS: Int,
+    MEMBERSHIP_ROUTING_MAX_ITEMS_LIMIT: Int,
+    MEMBERSHIP_ROUTING_MAX_GROUPS_LIST_LIMIT: Int,
+    RECORDSET_ROUTING_DEFAULT_MAX_ITEMS: Int,
+    ZONE_ROUTING_DEFAULT_MAX_ITEMS: Int,
+    ZONE_ROUTING_MAX_ITEMS_LIMIT: Int
+)
+object LimitsConfig {
+  implicit val configReader: ConfigReader[LimitsConfig] =
+    ConfigReader.forProduct7[LimitsConfig, Int, Int, Int, Int, Int, Int, Int](
+      "batchchange-routing-max-items-limit",
+      "membership-routing-default-max-items",
+      "membership-routing-max-items-limit",
+      "membership-routing-max-groups-list-limit",
+      "recordset-routing-default-max-items",
+      "zone-routing-default-max-items",
+      "zone-routing-max-items-limit"
+    ) {
+      case (
+          batchchange_routing_max_items_limit,
+          membership_routing_default_max_items,
+          membership_routing_max_items_limit,
+          membership_routing_max_groups_list_limit,
+          recordset_routing_default_max_items,
+          zone_routing_default_max_items,
+          zone_routing_max_items_limit
+          ) =>
+        LimitsConfig(
+          batchchange_routing_max_items_limit,
+          membership_routing_default_max_items,
+          membership_routing_max_items_limit,
+          membership_routing_max_groups_list_limit,
+          recordset_routing_default_max_items,
+          zone_routing_default_max_items,
+          zone_routing_max_items_limit
+        )
+    }
+
+}

modules/api/src/main/scala/vinyldns/api/config/VinylDNSConfig.scala

@@ -37,6 +37,7 @@ import scala.reflect.ClassTag
 
 final case class VinylDNSConfig(
     serverConfig: ServerConfig,
+    limitsconfig: LimitsConfig,
     httpConfig: HttpConfig,
     highValueDomainConfig: HighValueDomainConfig,
     manualReviewConfig: ManualReviewConfig,
@@ -80,6 +81,7 @@ object VinylDNSConfig {
 
     for {
       config <- IO.delay(ConfigFactory.load())
+      limitsconfig <- loadIO[LimitsConfig](config, "vinyldns.api.limits") //Added Limitsconfig to fetch data from the reference.config and pass to LimitsConfig.config
       serverConfig <- loadIO[ServerConfig](config, "vinyldns")
       batchChangeConfig <- loadIO[BatchChangeConfig](config, "vinyldns")
       backendConfigs <- loadIO[BackendConfigs](config, "vinyldns.backend")
@@ -98,6 +100,7 @@ object VinylDNSConfig {
         .map(GlobalAcls.apply)
     } yield VinylDNSConfig(
       serverConfig,
+      limitsconfig,
       httpConfig,
       hvdConfig,
       manualReviewConfig,

modules/api/src/main/scala/vinyldns/api/route/BatchChangeRouting.scala

@@ -18,6 +18,7 @@ package vinyldns.api.route
 
 import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.server.{RejectionHandler, Route, ValidationRejection}
+import vinyldns.api.config.LimitsConfig
 import org.slf4j.{Logger, LoggerFactory}
 import vinyldns.api.config.ManualReviewConfig
 import vinyldns.core.domain.batch._
@@ -25,6 +26,7 @@ import vinyldns.api.domain.batch._
 
 class BatchChangeRoute(
     batchChangeService: BatchChangeServiceAlgebra,
+    limitsConfig: LimitsConfig,
     val vinylDNSAuthenticator: VinylDNSAuthenticator,
     manualReviewConfig: ManualReviewConfig
 ) extends VinylDNSJsonProtocol
@@ -54,7 +56,7 @@ class BatchChangeRoute(
     case scnpd: ScheduledChangeNotDue => complete(StatusCodes.Forbidden, scnpd.message)
   }
 
-  final private val MAX_ITEMS_LIMIT: Int = 100
+  final private val MAX_ITEMS_LIMIT: Int = limitsConfig.BATCHCHANGE_ROUTING_MAX_ITEMS_LIMIT
 
   val batchChangeRoute: Route = {
     val standardBatchChangeRoutes = path("zones" / "batchrecordchanges") {

modules/api/src/main/scala/vinyldns/api/route/MembershipRouting.scala

@@ -19,6 +19,7 @@ package vinyldns.api.route
 import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.server._
 import org.slf4j.{Logger, LoggerFactory}
+import vinyldns.api.config.LimitsConfig
 import vinyldns.api.domain.membership._
 import vinyldns.api.domain.zone.NotAuthorizedError
 import vinyldns.api.route.MembershipJsonProtocol.{CreateGroupInput, UpdateGroupInput}
@@ -26,12 +27,15 @@ import vinyldns.core.domain.membership.{Group, LockStatus}
 
 class MembershipRoute(
     membershipService: MembershipServiceAlgebra,
+    limitsConfig: LimitsConfig,
     val vinylDNSAuthenticator: VinylDNSAuthenticator
 ) extends VinylDNSJsonProtocol
     with VinylDNSDirectives[Throwable] {
-  final private val DEFAULT_MAX_ITEMS: Int = 100
-  final private val MAX_ITEMS_LIMIT: Int = 1000
-  final private val MAX_GROUPS_LIST_LIMIT: Int = 1500
+
+  final private val DEFAULT_MAX_ITEMS: Int = limitsConfig.MEMBERSHIP_ROUTING_DEFAULT_MAX_ITEMS
+  final private val MAX_ITEMS_LIMIT: Int = limitsConfig.MEMBERSHIP_ROUTING_MAX_ITEMS_LIMIT
+  final private val MAX_GROUPS_LIST_LIMIT: Int =
+    limitsConfig.MEMBERSHIP_ROUTING_MAX_GROUPS_LIST_LIMIT
 
   def getRoutes: Route = membershipRoute
 

modules/api/src/main/scala/vinyldns/api/route/RecordSetRouting.scala

@@ -22,6 +22,7 @@ import akka.util.Timeout
 import org.slf4j.{Logger, LoggerFactory}
 import vinyldns.api.Interfaces._
 import vinyldns.api.domain.record.RecordSetServiceAlgebra
+import vinyldns.api.config.LimitsConfig
 import vinyldns.api.domain.zone._
 import vinyldns.core.domain.record.NameSort.NameSort
 import vinyldns.core.domain.record.RecordType.RecordType
@@ -56,6 +57,7 @@ case class ListRecordSetsByZoneResponse(
 
 class RecordSetRoute(
     recordSetService: RecordSetServiceAlgebra,
+    limitsConfig: LimitsConfig,
     val vinylDNSAuthenticator: VinylDNSAuthenticator
 ) extends VinylDNSJsonProtocol
     with VinylDNSDirectives[Throwable] {
@@ -64,7 +66,7 @@ class RecordSetRoute(
 
   def logger: Logger = LoggerFactory.getLogger(classOf[RecordSetRoute])
 
-  final private val DEFAULT_MAX_ITEMS: Int = 100
+  final private val DEFAULT_MAX_ITEMS: Int = limitsConfig.RECORDSET_ROUTING_DEFAULT_MAX_ITEMS
 
   // Timeout must be long enough to allow the cluster to form
   implicit val rsCmdTimeout: Timeout = Timeout(10.seconds)

modules/api/src/main/scala/vinyldns/api/route/VinylDNSService.scala

@@ -22,7 +22,7 @@ import cats.effect.IO
 import fs2.concurrent.SignallingRef
 import io.prometheus.client.CollectorRegistry
 import org.json4s.MappingException
-import vinyldns.api.config.VinylDNSConfig
+import vinyldns.api.config.{LimitsConfig, VinylDNSConfig}
 import vinyldns.api.domain.auth.AuthPrincipalProvider
 import vinyldns.api.domain.batch.BatchChangeServiceAlgebra
 import vinyldns.api.domain.membership.MembershipServiceAlgebra
@@ -57,6 +57,7 @@ object VinylDNSService {
 // $COVERAGE-OFF$
 class VinylDNSService(
     val membershipService: MembershipServiceAlgebra,
+    val limits: LimitsConfig,
     val processingDisabled: SignallingRef[IO, Boolean],
     val zoneService: ZoneServiceAlgebra,
     val healthService: HealthService,
@@ -84,13 +85,15 @@ class VinylDNSService(
     )
 
   val zoneRoute: Route =
-    new ZoneRoute(zoneService, vinylDNSAuthenticator, vinyldnsConfig.crypto).getRoutes
-  val recordSetRoute: Route = new RecordSetRoute(recordSetService, vinylDNSAuthenticator).getRoutes
+    new ZoneRoute(zoneService, limits, vinylDNSAuthenticator, vinyldnsConfig.crypto).getRoutes
+  val recordSetRoute: Route =
+    new RecordSetRoute(recordSetService, limits, vinylDNSAuthenticator).getRoutes
   val membershipRoute: Route =
-    new MembershipRoute(membershipService, vinylDNSAuthenticator).getRoutes
+    new MembershipRoute(membershipService, limits, vinylDNSAuthenticator).getRoutes
   val batchChangeRoute: Route =
     new BatchChangeRoute(
       batchChangeService,
+      limits,
       vinylDNSAuthenticator,
       vinyldnsConfig.manualReviewConfig
     ).getRoutes

modules/api/src/main/scala/vinyldns/api/route/ZoneRouting.scala

@@ -20,6 +20,7 @@ import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.server._
 import akka.util.Timeout
 import org.slf4j.{Logger, LoggerFactory}
+import vinyldns.api.config.LimitsConfig
 import vinyldns.api.domain.zone._
 import vinyldns.core.crypto.CryptoAlgebra
 import vinyldns.core.domain.zone._
@@ -31,6 +32,7 @@ case class ZoneRejected(zone: Zone, errors: List[String])
 
 class ZoneRoute(
     zoneService: ZoneServiceAlgebra,
+    limitsConfig: LimitsConfig,
     val vinylDNSAuthenticator: VinylDNSAuthenticator,
     crypto: CryptoAlgebra
 ) extends VinylDNSJsonProtocol
@@ -40,8 +42,8 @@ class ZoneRoute(
 
   def logger: Logger = LoggerFactory.getLogger(classOf[ZoneRoute])
 
-  final private val DEFAULT_MAX_ITEMS: Int = 100
-  final private val MAX_ITEMS_LIMIT: Int = 100
+  final private val DEFAULT_MAX_ITEMS: Int = limitsConfig.ZONE_ROUTING_DEFAULT_MAX_ITEMS
+  final private val MAX_ITEMS_LIMIT: Int = limitsConfig.ZONE_ROUTING_MAX_ITEMS_LIMIT
 
   // Timeout must be long enough to allow the cluster to form
   implicit val zoneCmdTimeout: Timeout = Timeout(10.seconds)

modules/api/src/test/scala/vinyldns/api/VinylDNSTestHelpers.scala

@@ -18,12 +18,7 @@ package vinyldns.api
 
 import com.comcast.ip4s.IpAddress
 import org.joda.time.DateTime
-import vinyldns.api.config.{
-  BatchChangeConfig,
-  HighValueDomainConfig,
-  ManualReviewConfig,
-  ScheduledChangesConfig
-}
+import vinyldns.api.config.{BatchChangeConfig, HighValueDomainConfig, LimitsConfig, ManualReviewConfig, ScheduledChangesConfig}
 import vinyldns.api.domain.batch.V6DiscoveryNibbleBoundaries
 import vinyldns.core.domain.record._
 import vinyldns.core.domain.zone._
@@ -73,6 +68,9 @@ trait VinylDNSTestHelpers {
 
   val batchChangeLimit = 1000
 
+  val testLimitConfig: LimitsConfig =
+    LimitsConfig(100,100,1000,1500,100,100,100)
+
   val batchChangeConfig: BatchChangeConfig =
     BatchChangeConfig(batchChangeLimit, sharedApprovedTypes, v6DiscoveryNibbleBoundaries)
 

modules/api/src/test/scala/vinyldns/api/route/BatchChangeRoutingSpec.scala

@@ -54,24 +54,28 @@ class BatchChangeRoutingSpec()
   val batchChangeRoute: Route =
     new BatchChangeRoute(
       TestBatchChangeService,
+      VinylDNSTestHelpers.testLimitConfig,
       new TestVinylDNSAuthenticator(okAuth),
       VinylDNSTestHelpers.manualReviewConfig
     ).getRoutes
   val notAuthRoute: Route =
     new BatchChangeRoute(
       TestBatchChangeService,
+      VinylDNSTestHelpers.testLimitConfig,
       new TestVinylDNSAuthenticator(notAuth),
       VinylDNSTestHelpers.manualReviewConfig
     ).getRoutes
   val supportUserRoute: Route =
     new BatchChangeRoute(
       TestBatchChangeService,
+      VinylDNSTestHelpers.testLimitConfig,
       new TestVinylDNSAuthenticator(supportUserAuth),
       VinylDNSTestHelpers.manualReviewConfig
     ).getRoutes
   val superUserRoute: Route =
     new BatchChangeRoute(
       TestBatchChangeService,
+      VinylDNSTestHelpers.testLimitConfig,
       new TestVinylDNSAuthenticator(superUserAuth),
       VinylDNSTestHelpers.manualReviewConfig
     ).getRoutes

modules/api/src/test/scala/vinyldns/api/route/MembershipRoutingSpec.scala

@@ -31,6 +31,7 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 import org.scalatest.BeforeAndAfterEach
 import vinyldns.api.Interfaces._
+import vinyldns.api.config.LimitsConfig
 import vinyldns.api.domain.membership._
 import vinyldns.api.domain.zone.NotAuthorizedError
 import vinyldns.api.route.MembershipJsonProtocol.{CreateGroupInput, UpdateGroupInput}
@@ -49,10 +50,12 @@ class MembershipRoutingSpec
     with BeforeAndAfterEach {
 
   val membershipService: MembershipService = mock[MembershipService]
+  val testLimitConfig: LimitsConfig =
+    LimitsConfig(100,100,1000,1500,100,100,100)
   val okAuthRoute: Route =
-    new MembershipRoute(membershipService, new TestVinylDNSAuthenticator(okAuth)).getRoutes
+    new MembershipRoute(membershipService,testLimitConfig, new TestVinylDNSAuthenticator(okAuth)).getRoutes
   val superUserRoute: Route =
-    new MembershipRoute(membershipService, new TestVinylDNSAuthenticator(superUserAuth)).getRoutes
+    new MembershipRoute(membershipService,testLimitConfig, new TestVinylDNSAuthenticator(superUserAuth)).getRoutes
   var membershipRoute: Route = _
 
   override protected def beforeEach(): Unit = {

modules/api/src/test/scala/vinyldns/api/route/RecordSetRoutingSpec.scala

@@ -26,6 +26,7 @@ import org.json4s.jackson.JsonMethods._
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 import vinyldns.api.Interfaces._
+import vinyldns.api.config.LimitsConfig
 import vinyldns.api.domain.record.{ListRecordSetChangesResponse, RecordSetServiceAlgebra}
 import vinyldns.api.domain.zone._
 import vinyldns.core.TestMembershipData.okAuth
@@ -638,8 +639,12 @@ class RecordSetRoutingSpec
   }
 
   val recordSetService: RecordSetServiceAlgebra = new TestService
+
+  val testLimitConfig: LimitsConfig =
+    LimitsConfig(100,100,1000,1500,100,100,100)
+
   val recordSetRoute: Route =
-    new RecordSetRoute(recordSetService, new TestVinylDNSAuthenticator(okAuth)).getRoutes
+    new RecordSetRoute(recordSetService,testLimitConfig, new TestVinylDNSAuthenticator(okAuth)).getRoutes
 
   private def rsJson(recordSet: RecordSet): String =
     compact(render(Extraction.decompose(recordSet)))

modules/api/src/test/scala/vinyldns/api/route/VinylDNSDirectivesSpec.scala

@@ -17,7 +17,6 @@
 package vinyldns.api.route
 
 import java.io.IOException
-
 import akka.http.scaladsl.model.{HttpEntity, HttpResponse, StatusCodes}
 import akka.http.scaladsl.server.{Directives, Route}
 import akka.http.scaladsl.testkit.ScalatestRouteTest
@@ -29,6 +28,7 @@ import org.scalatest.wordspec.AnyWordSpec
 import org.scalatest.{BeforeAndAfterEach, OneInstancePerTest}
 import org.scalatestplus.mockito.MockitoSugar
 import org.slf4j.{Logger, LoggerFactory}
+import vinyldns.api.config.LimitsConfig
 import vinyldns.api.domain.zone.ZoneServiceAlgebra
 import vinyldns.core.crypto.NoOpCrypto
 import vinyldns.core.route.Monitor
@@ -53,8 +53,11 @@ class VinylDNSDirectivesSpec
 
   def logger: Logger = LoggerFactory.getLogger(classOf[VinylDNSDirectivesSpec])
 
+  val testLimitConfig: LimitsConfig =
+    LimitsConfig(100,100,1000,1500,100,100,100)
+
   val zoneRoute: Route =
-    new ZoneRoute(mock[ZoneServiceAlgebra], mock[VinylDNSAuthenticator], NoOpCrypto.instance).getRoutes
+    new ZoneRoute(mock[ZoneServiceAlgebra],testLimitConfig, mock[VinylDNSAuthenticator], NoOpCrypto.instance).getRoutes
 
   val zoneService: ZoneServiceAlgebra = mock[ZoneServiceAlgebra]
 

modules/api/src/test/scala/vinyldns/api/route/ZoneRoutingSpec.scala

@@ -28,6 +28,7 @@ import org.scalatest.OneInstancePerTest
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 import vinyldns.api.Interfaces._
+import vinyldns.api.config.LimitsConfig
 import vinyldns.api.domain.zone.{ZoneServiceAlgebra, _}
 import vinyldns.core.TestMembershipData._
 import vinyldns.core.TestZoneData._
@@ -137,9 +138,11 @@ class ZoneRoutingSpec
       """secret = "8B06A7F3BC8A2497736F1916A123AA40E88217BE9264D8872597EF7A6E5DCE61""""
     )
   )
+  val testLimitConfig: LimitsConfig =
+    LimitsConfig(100,100,1000,1500,100,100,100)
 
   val zoneRoute: Route =
-    new ZoneRoute(TestZoneService, new TestVinylDNSAuthenticator(okAuth), crypto).getRoutes
+    new ZoneRoute(TestZoneService,testLimitConfig, new TestVinylDNSAuthenticator(okAuth), crypto).getRoutes
 
   object TestZoneService extends ZoneServiceAlgebra {
     def connectToZone(

modules/docs/src/main/mdoc/operator/config-api.md

@@ -636,7 +636,20 @@ v6-discovery-nibble-boundaries {
         }
      }
   }
-
+  
+  # limits for batchchange routing, membership routing , recordset routing , zone routing
+  api {
+  limits {
+  batchchange-routing-max-items-limit = 100
+  membership-routing-default-max-items = 100
+  membership-routing-max-items-limit = 1000
+  membership-routing-max-groups-list-limit = 1500
+  recordset-routing-default-max-items= 100
+  zone-routing-default-max-items = 100
+  zone-routing-max-items-limit = 100
+  }
+  }
+  
   # the DDNS connection information for the default dns backend
   defaultZoneConnection {
      name = "vinyldns."