From e03690271aa600044cdb39a571bdd3127116851e Mon Sep 17 00:00:00 2001 From: Paul Cleary Date: Thu, 23 May 2019 10:56:52 -0400 Subject: [PATCH] Wildcard search (#636) * Add wildcard support * Add wildcard search for zones * Update documentation * Fix for AWSAuthenticator to support asterisks --- .gitignore | 2 + .../recordsets/list_recordsets_test.py | 14 ++-- .../live_tests/zones/list_zones_test.py | 8 +-- .../api/route/Aws4Authenticator.scala | 1 + .../docs/src/main/tut/api/list-recordsets.md | 2 +- modules/docs/src/main/tut/api/list-zones.md | 2 +- modules/docs/src/main/tut/getting-help.md | 2 +- ...qlRecordSetRepositoryIntegrationSpec.scala | 56 ++++++++++++++-- .../MySqlZoneRepositoryIntegrationSpec.scala | 66 ++++++++++++++++++- .../repository/MySqlRecordSetRepository.scala | 2 +- .../repository/MySqlZoneRepository.scala | 2 +- modules/portal/.gitignore | 2 + 12 files changed, 136 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index fa9490200..b66f9d61a 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ release.version .ensime_cache package-lock.json *trustStore.jks +.bloop +.metals diff --git a/modules/api/functional_test/live_tests/recordsets/list_recordsets_test.py b/modules/api/functional_test/live_tests/recordsets/list_recordsets_test.py index 8d3ad1f2c..d4e5f2d06 100644 --- a/modules/api/functional_test/live_tests/recordsets/list_recordsets_test.py +++ b/modules/api/functional_test/live_tests/recordsets/list_recordsets_test.py @@ -214,7 +214,7 @@ def test_list_recordsets_with_record_name_filter_all(rs_fixture): client = rs_fixture.client ok_zone = rs_fixture.test_context - list_results = client.list_recordsets(ok_zone['id'], record_name_filter="list", status=200) + list_results = client.list_recordsets(ok_zone['id'], record_name_filter="*list*", status=200) rs_fixture.check_recordsets_page_accuracy(list_results, size=10, offset=0) @@ -227,7 +227,7 @@ def test_list_recordsets_with_record_name_filter_and_page_size(rs_fixture): ok_zone = rs_fixture.test_context #page of 4 items - list_results = client.list_recordsets(ok_zone['id'], max_items=4, record_name_filter="CNAME", status=200) + list_results = client.list_recordsets(ok_zone['id'], max_items=4, record_name_filter="*CNAME*", status=200) assert_that(list_results['recordSets'], has_length(4)) list_results_records = list_results['recordSets']; @@ -235,7 +235,7 @@ def test_list_recordsets_with_record_name_filter_and_page_size(rs_fixture): assert_that(list_results_records[i]['name'], contains_string('CNAME')) #page of 5 items but excess max items - list_results = client.list_recordsets(ok_zone['id'], max_items=7, record_name_filter="CNAME", status=200) + list_results = client.list_recordsets(ok_zone['id'], max_items=7, record_name_filter="*CNAME*", status=200) assert_that(list_results['recordSets'], has_length(5)) list_results_records = list_results['recordSets']; @@ -252,12 +252,12 @@ def test_list_recordsets_with_record_name_filter_and_chaining_pages_with_nextId( ok_zone = rs_fixture.test_context #page of 2 items - list_results = client.list_recordsets(ok_zone['id'], max_items=2, record_name_filter="CNAME", status=200) + list_results = client.list_recordsets(ok_zone['id'], max_items=2, record_name_filter="*CNAME*", status=200) assert_that(list_results['recordSets'], has_length(2)) start_key = list_results['nextId'] #page of 2 items - list_results = client.list_recordsets(ok_zone['id'], start_from=start_key, max_items=2, record_name_filter="CNAME", status=200) + list_results = client.list_recordsets(ok_zone['id'], start_from=start_key, max_items=2, record_name_filter="*CNAME*", status=200) assert_that(list_results['recordSets'], has_length(2)) list_results_records = list_results['recordSets']; @@ -272,7 +272,7 @@ def test_list_recordsets_with_record_name_filter_one(rs_fixture): client = rs_fixture.client ok_zone = rs_fixture.test_context - list_results = client.list_recordsets(ok_zone['id'], record_name_filter="8", status=200) + list_results = client.list_recordsets(ok_zone['id'], record_name_filter="*8*", status=200) rs_fixture.check_recordsets_page_accuracy(list_results, size=1, offset=8) @@ -283,7 +283,7 @@ def test_list_recordsets_with_record_name_filter_none(rs_fixture): client = rs_fixture.client ok_zone = rs_fixture.test_context - list_results = client.list_recordsets(ok_zone['id'], record_name_filter="Dummy", status=200) + list_results = client.list_recordsets(ok_zone['id'], record_name_filter="*Dummy*", status=200) rs_fixture.check_recordsets_page_accuracy(list_results, size=0, offset=0) diff --git a/modules/api/functional_test/live_tests/zones/list_zones_test.py b/modules/api/functional_test/live_tests/zones/list_zones_test.py index 679ca9b6f..26c2571ea 100644 --- a/modules/api/functional_test/live_tests/zones/list_zones_test.py +++ b/modules/api/functional_test/live_tests/zones/list_zones_test.py @@ -199,7 +199,7 @@ def test_list_zones_with_search_first_page(list_zones_context): """ Test that the first page of listing zones returns correctly when a name filter is provided """ - result = list_zones_context.client.list_zones(name_filter='searched', max_items=2, status=200) + result = list_zones_context.client.list_zones(name_filter='*searched*', max_items=2, status=200) zones = result['zones'] assert_that(zones, has_length(2)) @@ -208,7 +208,7 @@ def test_list_zones_with_search_first_page(list_zones_context): assert_that(result['nextId'], is_('list-zones-test-searched-2.')) assert_that(result['maxItems'], is_(2)) - assert_that(result['nameFilter'], is_('searched')) + assert_that(result['nameFilter'], is_('*searched*')) assert_that(result, is_not(has_key('startFrom'))) @@ -231,7 +231,7 @@ def test_list_zones_with_search_last_page(list_zones_context): """ Test that the second page of listing zones returns correctly when a name filter is provided """ - result = list_zones_context.client.list_zones(name_filter='searched', start_from="list-zones-test-searched-2.", max_items=2, status=200) + result = list_zones_context.client.list_zones(name_filter='*searched*', start_from="list-zones-test-searched-2.", max_items=2, status=200) zones = result['zones'] assert_that(zones, has_length(1)) @@ -239,5 +239,5 @@ def test_list_zones_with_search_last_page(list_zones_context): assert_that(result, is_not(has_key('nextId'))) assert_that(result['maxItems'], is_(2)) - assert_that(result['nameFilter'], is_('searched')) + assert_that(result['nameFilter'], is_('*searched*')) assert_that(result['startFrom'], is_('list-zones-test-searched-2.')) diff --git a/modules/api/src/main/scala/vinyldns/api/route/Aws4Authenticator.scala b/modules/api/src/main/scala/vinyldns/api/route/Aws4Authenticator.scala index b0af592fa..c93a185bd 100644 --- a/modules/api/src/main/scala/vinyldns/api/route/Aws4Authenticator.scala +++ b/modules/api/src/main/scala/vinyldns/api/route/Aws4Authenticator.scala @@ -239,6 +239,7 @@ class Aws4Authenticator { . // and doesn't encode '~' at all replaceAllLiterally("%7E", "~") + .replaceAllLiterally("*", "%2A") // aws encodes the asterisk specially private def hexString(bs: Array[Byte]) = bs.foldLeft("")((out, b) => f"$out%s${b & 0x0ff}%02x") diff --git a/modules/docs/src/main/tut/api/list-recordsets.md b/modules/docs/src/main/tut/api/list-recordsets.md index 81828ff0d..40de4c639 100755 --- a/modules/docs/src/main/tut/api/list-recordsets.md +++ b/modules/docs/src/main/tut/api/list-recordsets.md @@ -16,7 +16,7 @@ Retrieves a list of RecordSets from the zone name | type | required? | description | ------------ | ------------- | ----------- | :---------- | -recordNameFilter | string | no | One or more characters contained in the name of the record set to search for. For example `vinyl`. This is a contains search only, no wildcards or regular expressions are supported | +recordNameFilter | string | no | Characters that are part of the record name to search for. The wildcard character `*` is supported, for example `www*`. Omit the wildcard when searching for an exact record name. | startFrom | *any* | no | In order to advance through pages of results, the startFrom is set to the `nextId` that is returned on the previous response. It is up to the client to maintain previous pages if the client wishes to advance forward and backward. If not specified, will return the first page of results | maxItems | integer | no | The number of items to return in the page. Valid values are 1 to 100. Defaults to 100 if not provided. | diff --git a/modules/docs/src/main/tut/api/list-zones.md b/modules/docs/src/main/tut/api/list-zones.md index 0c9e012df..a747acc11 100755 --- a/modules/docs/src/main/tut/api/list-zones.md +++ b/modules/docs/src/main/tut/api/list-zones.md @@ -16,7 +16,7 @@ Retrieves the list of zones a user has access to. The zone name is only sorted name | type | required? | description | ------------ | ------------- | ----------- | :---------- | -nameFilter | string | no | One or more characters contained in the name of the zone to search for. For example `www-`. This is a contains search only, no wildcards or regular expressions are supported | +nameFilter | string | no | Characters that are part of the zone name to search for. The wildcard character `*` is supported, for example `www*`. Omit the wildcard character when searching for an exact zone name. | startFrom | *any* | no | In order to advance through pages of results, the startFrom is set to the `nextId` that is returned on the previous response. It is up to the client to maintain previous pages if the client wishes to advance forward and backward. If not specified, will return the first page of results | maxItems | int | no | The number of items to return in the page. Valid values are 1 - 100. Defaults to 100 if not provided. | diff --git a/modules/docs/src/main/tut/getting-help.md b/modules/docs/src/main/tut/getting-help.md index 9c2056df7..b1dcbc21a 100755 --- a/modules/docs/src/main/tut/getting-help.md +++ b/modules/docs/src/main/tut/getting-help.md @@ -7,7 +7,7 @@ position: 7 # Getting Help - Gitter community: - + - Contact the VinylDNS Core Team: vinyldns-core@googlegroups.com diff --git a/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlRecordSetRepositoryIntegrationSpec.scala b/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlRecordSetRepositoryIntegrationSpec.scala index 049daaa9d..a280b6c9a 100644 --- a/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlRecordSetRepositoryIntegrationSpec.scala +++ b/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlRecordSetRepositoryIntegrationSpec.scala @@ -341,10 +341,7 @@ class MySqlRecordSetRepositoryIntegrationSpec found.recordSets should contain theSameElementsInOrderAs existing.slice(2, 4) } "return the record sets after startFrom respecting maxItems and filter" in { - // load some deterministic names so we can filter and respect max items val recordNames = List("aaa", "bbb", "ccc", "ddd", "eeez", "fffz", "ggg", "hhhz", "iii", "jjj") - - // our search will be filtered by records with "z" val expectedNames = recordNames.filter(_.contains("z")) val newRecordSets = @@ -359,9 +356,58 @@ class MySqlRecordSetRepositoryIntegrationSpec val changes = newRecordSets.map(makeTestAddChange(_, okZone)) insert(changes) - // start after the second, pulling 3 records that have "z" val startFrom = Some(newRecordSets(1).name) - val found = repo.listRecordSets(okZone.id, startFrom, Some(3), Some("z")).unsafeRunSync() + val found = repo.listRecordSets(okZone.id, startFrom, Some(3), Some("*z*")).unsafeRunSync() + found.recordSets.map(_.name) should contain theSameElementsInOrderAs expectedNames + } + "return record sets using starts with wildcard" in { + val recordNames = List("aaa", "aab", "ccc") + val expectedNames = recordNames.filter(_.startsWith("aa")) + + val newRecordSets = + for { + n <- recordNames + } yield + aaaa.copy( + zoneId = okZone.id, + name = n, + id = UUID.randomUUID().toString) + + val changes = newRecordSets.map(makeTestAddChange(_, okZone)) + insert(changes) + + val found = repo.listRecordSets(okZone.id, None, Some(3), Some("aa*")).unsafeRunSync() + found.recordSets.map(_.name) should contain theSameElementsInOrderAs expectedNames + } + "return record sets using ends with wildcard" in { + val recordNames = List("aaa", "aab", "ccb") + val expectedNames = recordNames.filter(_.endsWith("b")) + + val newRecordSets = + for { + n <- recordNames + } yield aaaa.copy(zoneId = okZone.id, name = n, id = UUID.randomUUID().toString) + + val changes = newRecordSets.map(makeTestAddChange(_, okZone)) + insert(changes) + + val found = repo.listRecordSets(okZone.id, None, Some(3), Some("*b")).unsafeRunSync() + found.recordSets.map(_.name) should contain theSameElementsInOrderAs expectedNames + } + "return record sets exact match with no wildcards" in { + // load some deterministic names so we can filter and respect max items + val recordNames = List("aaa", "aab", "ccb") + val expectedNames = List("aaa") + + val newRecordSets = + for { + n <- recordNames + } yield aaaa.copy(zoneId = okZone.id, name = n, id = UUID.randomUUID().toString) + + val changes = newRecordSets.map(makeTestAddChange(_, okZone)) + insert(changes) + + val found = repo.listRecordSets(okZone.id, None, Some(3), Some("aaa")).unsafeRunSync() found.recordSets.map(_.name) should contain theSameElementsInOrderAs expectedNames } "pages through the list properly" in { diff --git a/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlZoneRepositoryIntegrationSpec.scala b/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlZoneRepositoryIntegrationSpec.scala index 1c0170bb4..d3369c4a1 100644 --- a/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlZoneRepositoryIntegrationSpec.scala +++ b/modules/mysql/src/it/scala/vinyldns/mysql/repository/MySqlZoneRepositoryIntegrationSpec.scala @@ -384,7 +384,7 @@ class MySqlZoneRepositoryIntegrationSpec val f = for { _ <- saveZones(testZones) - retrieved <- repo.listZones(superUserAuth, zoneNameFilter = Some("system")) + retrieved <- repo.listZones(superUserAuth, zoneNameFilter = Some("system*")) } yield retrieved f.unsafeRunSync().zones should contain theSameElementsAs expectedZones @@ -405,7 +405,69 @@ class MySqlZoneRepositoryIntegrationSpec val f = for { _ <- saveZones(testZones) - retrieved <- repo.listZones(auth, zoneNameFilter = Some("system")) + retrieved <- repo.listZones(auth, zoneNameFilter = Some("system*")) + } yield retrieved + + f.unsafeRunSync().zones should contain theSameElementsInOrderAs expectedZones + } + + "support starts with wildcard" in { + + val testZones = Seq( + testZone("system-test", adminGroupId = "foo"), + testZone("system-temp", adminGroupId = "foo"), + testZone("system-nomatch", adminGroupId = "bar") + ) + + val expectedZones = Seq(testZones(0), testZones(1)).sortBy(_.name) + + val auth = AuthPrincipal(dummyUser, Seq("foo")) + + val f = + for { + _ <- saveZones(testZones) + retrieved <- repo.listZones(auth, zoneNameFilter = Some("system*")) + } yield retrieved + + f.unsafeRunSync().zones should contain theSameElementsInOrderAs expectedZones + } + + "support ends with wildcard" in { + + val testZones = Seq( + testZone("system-test", adminGroupId = "foo"), + testZone("system-temp", adminGroupId = "foo"), + testZone("system-nomatch", adminGroupId = "bar") + ) + + val expectedZones = Seq(testZones(0)) + + val auth = AuthPrincipal(dummyUser, Seq("foo")) + + val f = + for { + _ <- saveZones(testZones) + retrieved <- repo.listZones(auth, zoneNameFilter = Some("*test")) + } yield retrieved + + f.unsafeRunSync().zones should contain theSameElementsInOrderAs expectedZones + } + + "support contains wildcard" in { + val testZones = Seq( + testZone("system-jokerswild", adminGroupId = "foo"), + testZone("system-wildcard", adminGroupId = "foo"), + testZone("system-nomatch", adminGroupId = "bar") + ) + + val expectedZones = Seq(testZones(0), testZones(1)) + + val auth = AuthPrincipal(dummyUser, Seq("foo")) + + val f = + for { + _ <- saveZones(testZones) + retrieved <- repo.listZones(auth, zoneNameFilter = Some("*wild*")) } yield retrieved f.unsafeRunSync().zones should contain theSameElementsInOrderAs expectedZones diff --git a/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlRecordSetRepository.scala b/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlRecordSetRepository.scala index 54b65684d..2fe6a0222 100644 --- a/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlRecordSetRepository.scala +++ b/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlRecordSetRepository.scala @@ -191,7 +191,7 @@ class MySqlRecordSetRepository extends RecordSetRepository with Monitored { val params = (Some('zoneId -> zoneId) ++ startFrom.map(n => 'startFrom -> n) ++ - recordNameFilter.map(f => 'nameFilter -> s"%$f%") ++ + recordNameFilter.map(f => 'nameFilter -> f.replace('*', '%')) ++ maxItems.map(m => 'maxItems -> m)).toSeq val query = "SELECT data FROM recordset WHERE zone_id = {zoneId} " + opts diff --git a/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlZoneRepository.scala b/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlZoneRepository.scala index d78fd10ec..95d68ebb4 100644 --- a/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlZoneRepository.scala +++ b/modules/mysql/src/main/scala/vinyldns/mysql/repository/MySqlZoneRepository.scala @@ -228,7 +228,7 @@ class MySqlZoneRepository extends ZoneRepository with ProtobufConversions with M sb.append(withAccessorCheck) val filters = List( - zoneNameFilter.map(flt => s"z.name LIKE '%$flt%'"), + zoneNameFilter.map(flt => s"z.name LIKE '${flt.replace('*', '%')}'"), startFrom.map(os => s"z.name > '$os'") ).flatten diff --git a/modules/portal/.gitignore b/modules/portal/.gitignore index 872a17b7a..de7280a39 100644 --- a/modules/portal/.gitignore +++ b/modules/portal/.gitignore @@ -12,3 +12,5 @@ package-lock.json release.version private public/gentelella +.bloop +.metals