mir/vinyldns
2
0
Fork 0
mirror of https://github.com/VinylDNS/vinyldns synced 2025-08-22 02:02:14 +00:00
Code Issues Releases Wiki Activity

Add zone sync scheduler

Browse Source
This commit is contained in:
Aravindh-Raju 2023-01-02 18:20:53 +05:30
parent 94277086a6
commit 1a9bf5cd89
No known key found for this signature in database
GPG Key ID: 6B4D566AC36626F6
21 changed files with 199 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/api/src/main/scala/vinyldns/api/Boot.scala
View File

@ -46,10 +46,15 @@ import scala.concurrent.{ExecutionContext, Future}
import scala.io.{Codec, Source}
import vinyldns.core.notifier.NotifierLoader
import vinyldns.core.repository.DataStoreLoader
import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit}
object Boot extends App {
private val logger = LoggerFactory.getLogger("Boot")
// Create a ScheduledExecutorService with a single thread
private val executor: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor()
private implicit val ec: ExecutionContext = scala.concurrent.ExecutionContext.global
private implicit val cs: ContextShift[IO] = IO.contextShift(ec)
private implicit val timer: Timer[IO] = IO.timer(ec)
@ -93,6 +98,11 @@ object Boot extends App {
repositories.userRepository
)
_ <- APIMetrics.initialize(vinyldnsConfig.apiMetricSettings)
// Schedule the task to be executed every 5 seconds
_ <- IO(executor.scheduleAtFixedRate(() => {
val zoneChanges = zoneSyncScheduleHandler.zoneSyncScheduler(repositories.zoneRepository).unsafeRunSync()
zoneChanges.foreach(zone => messageQueue.send(zone).unsafeRunSync())
}, 0, 5, TimeUnit.SECONDS))
_ <- CommandHandler.run(
messageQueue,
msgsPerPoll,

8
modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneChangeGenerator.scala
View File

@ -61,6 +61,14 @@ object ZoneChangeGenerator {
ZoneChangeStatus.Pending
)
def forSyncs(zone: Zone): ZoneChange =
ZoneChange(
zone.copy(updated = Some(Instant.now.truncatedTo(ChronoUnit.MILLIS)), status = ZoneStatus.Syncing),
zone.scheduleRequestor.get,
ZoneChangeType.Sync,
ZoneChangeStatus.Pending
)
def forDelete(zone: Zone, authPrincipal: AuthPrincipal): ZoneChange =
ZoneChange(
zone.copy(updated = Some(Instant.now.truncatedTo(ChronoUnit.MILLIS)), status = ZoneStatus.Deleted),

4
modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneProtocol.scala
View File

@ -44,6 +44,7 @@ case class ZoneInfo(
adminGroupName: String,
latestSync: Option[Instant],
backendId: Option[String],
recurrenceSchedule: Option[String],
accessLevel: AccessLevel
)
@ -70,6 +71,7 @@ object ZoneInfo {
adminGroupName = groupName,
latestSync = zone.latestSync,
backendId = zone.backendId,
recurrenceSchedule = zone.recurrenceSchedule,
accessLevel = accessLevel
)
}
@ -90,6 +92,7 @@ case class ZoneSummaryInfo(
adminGroupName: String,
latestSync: Option[Instant],
backendId: Option[String],
recurrenceSchedule: Option[String],
accessLevel: AccessLevel
)
@ -111,6 +114,7 @@ object ZoneSummaryInfo {
adminGroupName = groupName,
latestSync = zone.latestSync,
zone.backendId,
recurrenceSchedule = zone.recurrenceSchedule,
accessLevel = accessLevel
)
}

9
modules/api/src/main/scala/vinyldns/api/domain/zone/ZoneService.scala
View File

@ -28,6 +28,12 @@ import vinyldns.core.domain.zone._
import vinyldns.core.queue.MessageQueue
import vinyldns.core.domain.DomainHelpers.ensureTrailingDot
import vinyldns.core.domain.backend.BackendResolver
import com.cronutils.model.CronType
import com.cronutils.model.definition.{CronDefinition, CronDefinitionBuilder}
import com.cronutils.model.time.ExecutionTime
import com.cronutils.parser.CronParser
import java.time.{Instant, ZoneId}
import java.time.temporal.ChronoUnit
object ZoneService {
def apply(
@ -101,7 +107,8 @@ class ZoneService(
_ <- adminGroupExists(updateZoneInput.adminGroupId)
// if admin group changes, this confirms user has access to new group
_ <- canChangeZone(auth, updateZoneInput.name, updateZoneInput.adminGroupId).toResult
zoneWithUpdates = Zone(updateZoneInput, existingZone)
updatedZoneInput = if(updateZoneInput.recurrenceSchedule.isDefined) updateZoneInput.copy(scheduleRequestor = Some(auth.signedInUser.userName)) else updateZoneInput
zoneWithUpdates = Zone(updatedZoneInput, existingZone)
_ <- validateZoneConnectionIfChanged(zoneWithUpdates, existingZone)
updateZoneChange <- ZoneChangeGenerator
.forUpdate(zoneWithUpdates, existingZone, auth, crypto)

72
modules/api/src/main/scala/vinyldns/api/domain/zone/zoneSyncScheduleHandler.scala Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright 2018 Comcast Cable Communications Management, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package vinyldns.api.domain.zone
import cats.effect.IO
import com.cronutils.model.CronType
import com.cronutils.model.definition.{CronDefinition, CronDefinitionBuilder}
import com.cronutils.model.time.ExecutionTime
import com.cronutils.parser.CronParser
import vinyldns.core.domain.zone.{Zone, ZoneChange, ZoneRepository}
import java.time.{Instant, ZoneId}
import java.time.temporal.ChronoUnit
object zoneSyncScheduleHandler {
// Define the function you want to repeat
def zoneSyncScheduler(zoneRepository: ZoneRepository): IO[Set[ZoneChange]] = {
for {
zones <- zoneRepository.getAllZonesWithSyncSchedule
zoneScheduleIds = getZoneWithSchedule(zones.toList)
zoneChanges <- getZoneChanges(zoneRepository, zoneScheduleIds)
} yield zoneChanges
}
def getZoneChanges(zoneRepository: ZoneRepository, zoneScheduleIds: List[String]): IO[Set[ZoneChange]] = {
if(zoneScheduleIds.nonEmpty) {
for{
getZones <- zoneRepository.getZones(zoneScheduleIds.toSet)
syncZoneChange = getZones.map(zone => ZoneChangeGenerator.forSyncs(zone))
} yield syncZoneChange
} else {
IO(Set.empty)
}
}
def getZoneWithSchedule(zone: List[Zone]): List[String] = {
var zonesWithSchedule: List[String] = List.empty
for(z <- zone) {
if (z.recurrenceSchedule.isDefined) {
val now = Instant.now()
val cronDefinition: CronDefinition = CronDefinitionBuilder.instanceDefinitionFor(CronType.QUARTZ)
val parser: CronParser = new CronParser(cronDefinition)
val executionTime: ExecutionTime = ExecutionTime.forCron(parser.parse(z.recurrenceSchedule.get))
val nextExecution = executionTime.nextExecution(now.atZone(ZoneId.systemDefault())).get()
val diff = ChronoUnit.SECONDS.between(now, nextExecution)
if (diff <= 5) {
zonesWithSchedule = zonesWithSchedule :+ z.id
} else {
List.empty
}
} else {
List.empty
}
}
zonesWithSchedule
}
}

8
modules/api/src/main/scala/vinyldns/api/route/DnsJsonProtocol.scala
View File

@ -111,7 +111,9 @@ trait DnsJsonProtocol extends JsonValidation {
(js \ "shared").default[Boolean](false),
(js \ "acl").default[ZoneACL](ZoneACL()),
(js \ "adminGroupId").required[String]("Missing Zone.adminGroupId"),
(js \ "backendId").optional[String]
(js \ "backendId").optional[String],
(js \ "recurrenceSchedule").optional[String],
(js \ "scheduleRequestor").optional[String],
).mapN(CreateZoneInput.apply)
}
@ -128,7 +130,9 @@ trait DnsJsonProtocol extends JsonValidation {
(js \ "shared").default[Boolean](false),
(js \ "acl").default[ZoneACL](ZoneACL()),
(js \ "adminGroupId").required[String]("Missing Zone.adminGroupId"),
(js \ "backendId").optional[String]
(js \ "recurrenceSchedule").optional[String],
(js \ "scheduleRequestor").optional[String],
(js \ "backendId").optional[String],
).mapN(UpdateZoneInput.apply)
}

2
modules/core/src/main/protobuf/VinylDNSProto.proto
View File

@ -49,6 +49,8 @@ message Zone {
optional int64 latestSync = 13;
optional bool isTest = 14 [default = false];
optional string backendId = 15;
optional string recurrenceSchedule = 16;
optional string scheduleRequestor = 17;
}
message AData {

18
modules/core/src/main/scala/vinyldns/core/domain/zone/Zone.scala
View File

@ -47,6 +47,8 @@ final case class Zone(
shared: Boolean = false,
acl: ZoneACL = ZoneACL(),
adminGroupId: String = "system",
recurrenceSchedule: Option[String] = None,
scheduleRequestor: Option[String] = None,
latestSync: Option[Instant] = None,
isTest: Boolean = false,
backendId: Option[String] = None
@ -75,6 +77,8 @@ final case class Zone(
sb.append("reverse=\"").append(isReverse).append("\"; ")
sb.append("isTest=\"").append(isTest).append("\"; ")
sb.append("created=\"").append(created).append("\"; ")
recurrenceSchedule.map(sb.append("recurrenceSchedule=\"").append(_).append("\"; "))
scheduleRequestor.map(sb.append("scheduleRequestor=\"").append(_).append("\"; "))
updated.map(sb.append("updated=\"").append(_).append("\"; "))
latestSync.map(sb.append("latestSync=\"").append(_).append("\"; "))
sb.append("]")
@ -95,7 +99,9 @@ object Zone {
acl = acl,
adminGroupId = adminGroupId,
backendId = backendId,
isTest = isTest
isTest = isTest,
recurrenceSchedule = recurrenceSchedule,
scheduleRequestor = scheduleRequestor
)
}
@ -110,7 +116,9 @@ object Zone {
shared = shared,
acl = acl,
adminGroupId = adminGroupId,
backendId = backendId
backendId = backendId,
recurrenceSchedule = recurrenceSchedule,
scheduleRequestor = scheduleRequestor
)
}
}
@ -123,7 +131,9 @@ final case class CreateZoneInput(
shared: Boolean = false,
acl: ZoneACL = ZoneACL(),
adminGroupId: String,
backendId: Option[String] = None
backendId: Option[String] = None,
recurrenceSchedule: Option[String] = None,
scheduleRequestor: Option[String] = None
)
final case class UpdateZoneInput(
@ -135,6 +145,8 @@ final case class UpdateZoneInput(
shared: Boolean = false,
acl: ZoneACL = ZoneACL(),
adminGroupId: String,
recurrenceSchedule: Option[String] = None,
scheduleRequestor: Option[String] = None,
backendId: Option[String] = None
)

2
modules/core/src/main/scala/vinyldns/core/domain/zone/ZoneRepository.scala
View File

@ -29,6 +29,8 @@ trait ZoneRepository extends Repository {
def getZones(zoneId: Set[String]): IO[Set[Zone]]
def getAllZonesWithSyncSchedule: IO[Set[Zone]]
def getZoneByName(zoneName: String): IO[Option[Zone]]
def getZonesByNames(zoneNames: Set[String]): IO[Set[Zone]]

6
modules/core/src/main/scala/vinyldns/core/protobuf/ProtobufConversions.scala
View File

@ -125,7 +125,9 @@ trait ProtobufConversions {
adminGroupId = zn.getAdminGroupId,
latestSync = if (zn.hasLatestSync) Some(Instant.ofEpochMilli(zn.getLatestSync)) else None,
isTest = zn.getIsTest,
backendId = if (zn.hasBackendId) Some(zn.getBackendId) else None
backendId = if (zn.hasBackendId) Some(zn.getBackendId) else None,
recurrenceSchedule = if (zn.hasRecurrenceSchedule) Some(zn.getRecurrenceSchedule) else None,
scheduleRequestor = if (zn.hasScheduleRequestor) Some(zn.getScheduleRequestor) else None
)
}
@ -401,6 +403,8 @@ trait ProtobufConversions {
zone.transferConnection.foreach(cn => builder.setTransferConnection(toPB(cn)))
zone.latestSync.foreach(dt => builder.setLatestSync(dt.toEpochMilli))
zone.backendId.foreach(bid => builder.setBackendId(bid))
zone.recurrenceSchedule.foreach(rs => builder.setRecurrenceSchedule(rs))
zone.scheduleRequestor.foreach(rs => builder.setScheduleRequestor(rs))
builder.build()
}

5
modules/mysql/src/main/resources/db/migration/V3.28__ZoneSchedule.sql Normal file
View File

@ -0,0 +1,5 @@
CREATE SCHEMA IF NOT EXISTS ${dbName};
USE ${dbName};
ALTER TABLE zone ADD zone_sync_schedule VARCHAR(256) NULL;

27
modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlZoneRepository.scala
View File

@ -48,10 +48,11 @@ class MySqlZoneRepository extends ZoneRepository with ProtobufConversions with M
*/
private final val PUT_ZONE =
sql"""
|INSERT INTO zone(id, name, admin_group_id, data)
| VALUES ({id}, {name}, {adminGroupId}, {data}) ON DUPLICATE KEY
|INSERT INTO zone(id, name, admin_group_id, zone_sync_schedule, data)
| VALUES ({id}, {name}, {adminGroupId}, {recurrenceSchedule}, {data}) ON DUPLICATE KEY
| UPDATE name=VALUES(name),
| admin_group_id=VALUES(admin_group_id),
| zone_sync_schedule=VALUES(zone_sync_schedule),
| data=VALUES(data);
""".stripMargin
@ -116,6 +117,13 @@ class MySqlZoneRepository extends ZoneRepository with ProtobufConversions with M
| FROM zone
""".stripMargin
private final val BASE_GET_ALL_ZONES_SQL =
"""
|SELECT data
| FROM zone
| WHERE zone_sync_schedule IS NOT NULL
""".stripMargin
private final val GET_ZONE_ACCESS_BY_ADMIN_GROUP_ID =
sql"""
|SELECT zone_id
@ -207,6 +215,19 @@ class MySqlZoneRepository extends ZoneRepository with ProtobufConversions with M
}
}
def getAllZonesWithSyncSchedule: IO[Set[Zone]] =
monitor("repo.ZoneJDBC.getAllZonesWithSyncSchedule") {
IO {
DB.readOnly { implicit s =>
SQL(
BASE_GET_ALL_ZONES_SQL
).map(extractZone(1))
.list()
.apply()
}.toSet
}
}
def getZonesByFilters(zoneNames: Set[String]): IO[Set[Zone]] =
if (zoneNames.isEmpty) {
IO.pure(Set())
@ -414,6 +435,7 @@ class MySqlZoneRepository extends ZoneRepository with ProtobufConversions with M
'id -> zone.id,
'name -> zone.name,
'adminGroupId -> zone.adminGroupId,
'recurrenceSchedule -> zone.recurrenceSchedule,
'data -> toPB(zone).toByteArray
): _*
)
@ -475,6 +497,7 @@ class MySqlZoneRepository extends ZoneRepository with ProtobufConversions with M
def saveTx(zone: Zone): IO[Either[DuplicateZoneError, Zone]] =
monitor("repo.ZoneJDBC.save") {
IO {
println(zone.recurrenceSchedule)
DB.localTx { implicit s =>
getZoneByNameInSession(zone.name) match {
case Some(foundZone) if zone.id != foundZone.id => DuplicateZoneError(zone.name).asLeft

2
modules/portal/Gruntfile.js
View File

@ -35,10 +35,12 @@ module.exports = function(grunt) {
{expand: true, flatten: true, src: ['node_modules/jquery/dist/jquery.min.js'], dest: 'public/js'},
{expand: true, flatten: true, src: ['node_modules/moment/min/moment.min.js'], dest: 'public/js'},
{expand: true, flatten: true, src: ['node_modules/jquery-ui-dist/jquery-ui.js'], dest: 'public/js'},
{expand: true, flatten: true, src: ['node_modules/angular-cron-jobs/dist/angular-cron-jobs.min.js'], dest: 'public/js'},
{expand: true, flatten: true, src: ['node_modules/bootstrap/dist/css/bootstrap.min.css'], dest: 'public/css'},
{expand: true, flatten: true, src: ['node_modules/font-awesome/css/font-awesome.min.css'], dest: 'public/css'},
{expand: true, flatten: true, src: ['node_modules/jquery-ui-dist/jquery-ui.css'], dest: 'public/css'},
{expand: true, flatten: true, src: ['node_modules/angular-cron-jobs/dist/angular-cron-jobs.min.css'], dest: 'public/css'},
// We're picking just the resources we need from the gentelella UI framework and temporarily storing them in mapped/ui/
{expand: true, flatten: true, cwd: 'node_modules/gentelella', dest: 'mapped/ui', src: '**/jquery.{smartWizard,dataTables.min,mousewheel.min}.js'},

3
modules/portal/app/controllers/FrontendController.scala
View File

@ -71,7 +71,8 @@ class FrontendController @Inject() (
}
def viewZone(zoneId: String): Action[AnyContent] = userAction.async { implicit request =>
Future(Ok(views.html.zones.zoneDetail(request.user.userName, zoneId)))
val canReview = request.user.isSuper || request.user.isSupport
Future(Ok(views.html.zones.zoneDetail(request.user.userName, canReview, zoneId)))
}
def viewRecordSets(): Action[AnyContent] = userAction.async { implicit request =>

3
modules/portal/app/views/main.scala.html
View File

@ -21,7 +21,7 @@
<link rel="stylesheet" type="text/css" href="/public/css/ui.css" />
<link rel="stylesheet" type="text/css" id="custom" href="/public/css/theme-overrides.css"/>
<link rel="stylesheet" type="text/css" id="custom" href="/public/css/vinyldns.css"/>
<link rel="stylesheet" type="text/css" href="/public/css/jquery-ui.css">
<link rel="stylesheet" type="text/css" href="/public/css/angular-cron-jobs.min.css">
<!-- EOF CSS INCLUDE -->
</head>
@ -158,6 +158,7 @@
<script src="/public/js/vinyldns.js"></script>
<script src="/public/app.js"></script>
<script src="/public/js/custom.js"></script>
<script src="/public/js/angular-cron-jobs.min.js"></script>
@pagePlugins

4
modules/portal/app/views/zones/zoneDetail.scala.html
View File

@ -1,4 +1,4 @@
@(rootAccountName: String, zoneId: String)(implicit request: play.api.mvc.Request[Any], customLinks: models.CustomLinks, meta: models.Meta)
@(rootAccountName: String, rootAccountCanReview: Boolean, zoneId: String)(implicit request: play.api.mvc.Request[Any], customLinks: models.CustomLinks, meta: models.Meta)
@import zoneTabs._
@content = {
@ -50,7 +50,7 @@
@manageRecords(request, meta)
</div>
<div class="tab-pane" id="tab2" ng-controller="ManageZonesController">
@manageZone(request, meta)
@manageZone(rootAccountCanReview, request, meta)
</div>
<div class="tab-pane" id="tab3">
@changeHistory(request)

13
modules/portal/app/views/zones/zoneTabs/manageZone.scala.html
View File

@ -1,4 +1,4 @@
@(implicit request: play.api.mvc.Request[Any], meta: models.Meta)
@(implicit rootAccountCanReview: Boolean, request: play.api.mvc.Request[Any], meta: models.Meta)
<div class="alert-wrapper">
<div ng-repeat="alert in alerts">
@ -233,6 +233,17 @@
</div>
</div>
<div class="col-md-15">
<div class="block">
<div class="col-md-9">
@if(rootAccountCanReview) {
<h4>Schedule Zone Sync</h4>
<cron-selection class="col-md-12" ng-model="updateZoneInfo.recurrenceSchedule" config="myZoneSyncScheduleConfig"></cron-selection>
}
</div>
</div>
</div>
</div>
</div>

1
modules/portal/karma.conf.js
View File

@ -21,6 +21,7 @@ module.exports = function(config) {
'js/angular.min.js',
'js/moment.min.js',
'js/ui.js',
'js/angular-cron-jobs.min.js',
'test_frameworks/*.js',
'js/vinyldns.js',
'lib/services/**/*.spec.js',

3
modules/portal/package.json
View File

@ -32,7 +32,8 @@
"karma-phantomjs-launcher": "^1.0.0",
"karma-spec-reporter": "^0.0.26",
"malihu-custom-scrollbar-plugin": "^3.1.5",
"phantomjs-prebuilt": "^2.1.16"
"phantomjs-prebuilt": "^2.1.16",
"angular-cron-jobs": "^3.2.1"
},
"scripts": {
"test": "unit"

13
modules/portal/public/lib/controllers/controller.manageZones.js
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
angular.module('controller.manageZones', [])
angular.module('controller.manageZones', ['angular-cron-jobs'])
.controller('ManageZonesController', function ($scope, $timeout, $log, recordsService, zonesService, groupsService,
profileService, utilityService, pagingService) {
@ -94,6 +94,15 @@ angular.module('controller.manageZones', [])
$("#delete_zone_connection_modal").modal("show");
};
$scope.myZoneSyncScheduleConfig = {
allowMultiple: true,
quartz: true,
options: {
allowMinute : false,
allowHour : false
}
}
$scope.submitDeleteZone = function() {
zonesService.delZone($scope.zoneInfo.id)
.then(function (response) {
@ -278,6 +287,8 @@ angular.module('controller.manageZones', [])
$scope.updateZoneInfo = angular.copy($scope.zoneInfo);
$scope.updateZoneInfo.hiddenKey = '';
$scope.updateZoneInfo.hiddenTransferKey = '';
$log.log('recordsService::getZone-success schedule: ', $scope.updateZoneInfo.recurrenceSchedule);
$log.log('recordsService::getZone-success: ', $scope.zoneInfo);
$scope.currentManageZoneState = $scope.manageZoneState.UPDATE;
$scope.refreshAclRuleDisplay();
$scope.refreshZoneChange();

3
project/Dependencies.scala
View File

@ -52,7 +52,8 @@ object Dependencies {
"com.sun.mail" % "javax.mail" % "1.6.2",
"javax.mail" % "javax.mail-api" % "1.6.2",
"com.amazonaws" % "aws-java-sdk-sns" % awsV withSources(),
"co.elastic.logging" % "logback-ecs-encoder" % "1.3.2"
"co.elastic.logging" % "logback-ecs-encoder" % "1.3.2",
"com.cronutils" % "cron-utils" % "9.1.6"
)
lazy val coreDependencies = Seq(