From 348bf3ef340d8e489f74fcaa9253ae4141d73b02 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Mon, 25 Nov 2024 13:20:42 -0500 Subject: [PATCH 01/22] Handle child broker to parent broker migration --- .../Operations/DataBrokerOperation.swift | 15 ++++++++++++++- .../DataBrokerProfileQueryOperationManager.swift | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index a04ab7d0b4..b729854ec8 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -131,7 +131,7 @@ class DataBrokerOperation: Operation, @unchecked Sendable { if let priorityDate = priorityDate { filteredAndSortedOperationsData = operationsData - .filter { $0.preferredRunDate != nil && $0.preferredRunDate! <= priorityDate } + .filtered(by: priorityDate) .sorted { $0.preferredRunDate! < $1.preferredRunDate! } } else { filteredAndSortedOperationsData = operationsData @@ -215,3 +215,16 @@ class DataBrokerOperation: Operation, @unchecked Sendable { } } // swiftlint:enable explicit_non_final_class + +extension Array where Element == BrokerJobData { + /// Filters jobs based on their preferred run date: + /// - Jobs with no preferred run date are included. + /// - Jobs with a preferred run date on or before the priority date are included. + /// + /// Note: Jobs without a preferred run date may be: + /// 1. From child brokers (skipped during runOptOutOperation) + /// 2. From former child brokers now acting as parent brokers (processed if extractedProfile hasn't been removed) + func filtered(by priorityDate: Date) -> [BrokerJobData] { + filter { $0.preferredRunDate == nil || $0.preferredRunDate! <= priorityDate } + } +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift index 6bb37ceb99..eef9280e4c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift @@ -291,11 +291,11 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager { } guard extractedProfile.removedDate == nil else { - Logger.dataBrokerProtection.debug("Profile already extracted, skipping...") + Logger.dataBrokerProtection.debug("Profile already removed, skipping...") return } - guard let optOutStep = brokerProfileQueryData.dataBroker.optOutStep(), optOutStep.optOutType != .parentSiteOptOut else { + guard !brokerProfileQueryData.dataBroker.performsOptOutWithinParent() else { Logger.dataBrokerProtection.debug("Broker opts out in parent, skipping...") return } From 9dfa64da907cc8a1927c78caf81dedf47c53f5db Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 26 Nov 2024 22:26:11 -0500 Subject: [PATCH 02/22] Fix operation data sorting --- .../Operations/DataBrokerOperation.swift | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index b729854ec8..5147e1d014 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -131,8 +131,8 @@ class DataBrokerOperation: Operation, @unchecked Sendable { if let priorityDate = priorityDate { filteredAndSortedOperationsData = operationsData - .filtered(by: priorityDate) - .sorted { $0.preferredRunDate! < $1.preferredRunDate! } + .filtered(using: priorityDate) + .sortedByPreferredRunDate() } else { filteredAndSortedOperationsData = operationsData } @@ -224,7 +224,22 @@ extension Array where Element == BrokerJobData { /// Note: Jobs without a preferred run date may be: /// 1. From child brokers (skipped during runOptOutOperation) /// 2. From former child brokers now acting as parent brokers (processed if extractedProfile hasn't been removed) - func filtered(by priorityDate: Date) -> [BrokerJobData] { + func filtered(using priorityDate: Date) -> [BrokerJobData] { filter { $0.preferredRunDate == nil || $0.preferredRunDate! <= priorityDate } } + + func sortedByPreferredRunDate() -> [BrokerJobData] { + sorted { lhs, rhs in + switch (lhs.preferredRunDate, rhs.preferredRunDate) { + case (nil, nil): + return false + case (nil, _): + return true + case (_, nil): + return false + case (let lhsRunDate?, let rhsRunDate?): + return lhsRunDate < rhsRunDate + } + } + } } From 56a777b67a5583ce1571ca8c30e38a6415840489 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 26 Nov 2024 22:26:25 -0500 Subject: [PATCH 03/22] Update kwold JSON --- .../Resources/JSON/kwold.com.json | 209 ++++++++++++------ 1 file changed, 140 insertions(+), 69 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json index cbe6abc734..b90a1fae58 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json @@ -1,76 +1,147 @@ { - "name": "Kwold", - "url": "kwold.com", - "version": "0.4.0", - "parent": "verecor.com", - "addedDatetime": 1702965600000, - "optOutUrl": "https://kwold.com/ns/control/privacy", - "steps": [ - { - "stepType": "scan", - "scanType": "templatedUrl", - "actions": [ + "name": "Kwold", + "url": "kwold.com", + "version": "0.5.0", + "addedDatetime": 1702965600000, + "optOutUrl": "https://kwold.com/ns/control/privacy", + "steps": [ { - "actionType": "navigate", - "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", - "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", - "ageRange": [ - "18-30", - "31-40", - "41-50", - "51-60", - "61-70", - "71-80", - "81+" - ] + "stepType": "scan", + "scanType": "templatedUrl", + "actions": [ + { + "actionType": "navigate", + "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", + "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", + "ageRange": [ + "18-30", + "31-40", + "41-50", + "51-60", + "61-70", + "71-80", + "81+" + ] + }, + { + "actionType": "extract", + "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", + "selector": ".card", + "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", + "profile": { + "name": { + "selector": ".card-title", + "beforeText": " ~" + }, + "alternativeNamesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "age": { + "beforeText": "years old", + "selector": ".card-title", + "afterText": " ~" + }, + "addressCityStateList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "relativesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", + "beforeText": ",", + "findElements": true + }, + "profileUrl": { + "selector": "a", + "identifierType": "path", + "identifier": "https://kwold.com/pp/${id}" + } + } + } + ] }, { - "actionType": "extract", - "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", - "selector": ".card", - "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", - "profile": { - "name": { - "selector": ".card-title", - "beforeText": " ~" - }, - "alternativeNamesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "age": { - "beforeText": "years old", - "selector": ".card-title", - "afterText": " ~" - }, - "addressCityStateList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "relativesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", - "beforeText": ",", - "findElements": true - }, - "profileUrl": { - "selector": "a", - "identifierType": "path", - "identifier": "https://kwold.com/pp/${id}" - } - } + "stepType": "optOut", + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://kwold.com/ns/control/privacy", + "id": "037f7920-b9e7-4214-a937-171ec641d641" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "5b9de12f-a52e-4bd0-b6ac-6884377d309b" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "48e5e7a8-af33-4629-a849-2cf926a518a3" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "bc2d26dc-3eef-478a-a04b-5671a1dbdf8b" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "7f2a685e-ddad-4c5a-8e80-a6d3a690851f" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "3a8a6e9d-c9a0-4e59-a8a4-fe4a05f3ce68" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "93ccf84a-a5ce-4dcf-8a78-143610723488" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" + } + ] } - ] - }, - { - "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + ], + "schedulingConfig": { + "retryError": 48, + "confirmOptOutScan": 72, + "maintenanceScan": 120, + "maxAttempts": -1 } - ], - "schedulingConfig": { - "retryError": 48, - "confirmOptOutScan": 72, - "maintenanceScan": 120, - "maxAttempts": -1 - } } From ee69d21c96f8207c0dac7f5fac57b2388a4afd51 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 26 Nov 2024 22:29:52 -0500 Subject: [PATCH 04/22] Refactor --- .../Operations/DataBrokerOperation.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index 5147e1d014..5e161e924f 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -228,14 +228,17 @@ extension Array where Element == BrokerJobData { filter { $0.preferredRunDate == nil || $0.preferredRunDate! <= priorityDate } } + /// Sorts BrokerJobData array based on their preferred run dates. + /// - Jobs with non-nil preferred run dates are sorted in ascending order (earliest date first). + /// - Jobs with nil preferred run dates come last, maintaining their original relative order. func sortedByPreferredRunDate() -> [BrokerJobData] { sorted { lhs, rhs in switch (lhs.preferredRunDate, rhs.preferredRunDate) { case (nil, nil): return false - case (nil, _): - return true case (_, nil): + return true + case (nil, _): return false case (let lhsRunDate?, let rhsRunDate?): return lhsRunDate < rhsRunDate From 70551e07e0f028e8e6caf05660ff6059fbd789ea Mon Sep 17 00:00:00 2001 From: Anh Do Date: Wed, 27 Nov 2024 01:07:04 -0500 Subject: [PATCH 05/22] Add test cases --- .../Operations/DataBrokerOperation.swift | 22 +++-- .../DataBrokerProtectionStatsPixels.swift | 4 + .../DataBrokerOperationTests.swift | 99 +++++++++++++++++++ .../DataBrokerProtectionTests/Mocks.swift | 7 ++ 4 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index 5e161e924f..d24eefb993 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -115,7 +115,7 @@ class DataBrokerOperation: Operation, @unchecked Sendable { } } - private func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] { + static func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] { let operationsData: [BrokerJobData] switch operationType { @@ -152,9 +152,9 @@ class DataBrokerOperation: Operation, @unchecked Sendable { let brokerProfileQueriesData = allBrokerProfileQueryData.filter { $0.dataBroker.id == dataBrokerID } - let filteredAndSortedOperationsData = filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData, - operationType: operationType, - priorityDate: priorityDate) + let filteredAndSortedOperationsData = Self.filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData, + operationType: operationType, + priorityDate: priorityDate) Logger.dataBrokerProtection.debug("filteredAndSortedOperationsData count: \(filteredAndSortedOperationsData.count, privacy: .public) for brokerID \(self.dataBrokerID, privacy: .public)") @@ -218,19 +218,25 @@ class DataBrokerOperation: Operation, @unchecked Sendable { extension Array where Element == BrokerJobData { /// Filters jobs based on their preferred run date: - /// - Jobs with no preferred run date are included. + /// - Opt-out jobs with no preferred run date are included. /// - Jobs with a preferred run date on or before the priority date are included. /// - /// Note: Jobs without a preferred run date may be: + /// Note: Opt-out jobs without a preferred run date may be: /// 1. From child brokers (skipped during runOptOutOperation) /// 2. From former child brokers now acting as parent brokers (processed if extractedProfile hasn't been removed) func filtered(using priorityDate: Date) -> [BrokerJobData] { - filter { $0.preferredRunDate == nil || $0.preferredRunDate! <= priorityDate } + filter { jobData in + guard let preferredRunDate = jobData.preferredRunDate else { + return jobData is OptOutJobData + } + + return preferredRunDate <= priorityDate + } } /// Sorts BrokerJobData array based on their preferred run dates. /// - Jobs with non-nil preferred run dates are sorted in ascending order (earliest date first). - /// - Jobs with nil preferred run dates come last, maintaining their original relative order. + /// - Opt-out jobs with nil preferred run dates come last, maintaining their original relative order. func sortedByPreferredRunDate() -> [BrokerJobData] { sorted { lhs, rhs in switch (lhs.preferredRunDate, rhs.preferredRunDate) { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift index 7534b98ee9..22798d7eb3 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift @@ -172,6 +172,10 @@ extension Date { static func nowMinus(hours: Int) -> Date { Calendar.current.date(byAdding: .hour, value: -hours, to: Date()) ?? Date() } + + static func nowPlus(hours: Int) -> Date { + nowMinus(hours: -hours) + } } final class DataBrokerProtectionStatsPixels: StatsPixels { diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift new file mode 100644 index 0000000000..bde629a234 --- /dev/null +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift @@ -0,0 +1,99 @@ +// +// DataBrokerOperationTests.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +@testable import DataBrokerProtection +import XCTest + +final class DataBrokerOperationTests: XCTestCase { + lazy var mockOptOutQueryData: [BrokerProfileQueryData] = { + let brokerId: Int64 = 1 + + let mockChildBrokerQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: nil, optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: nil)]) + } + let mockPastQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowMinus(hours: $0))]) + } + let mockFutureQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowPlus(hours: $0))]) + } + + return mockChildBrokerQueryData + mockPastQueryData + mockFutureQueryData + }() + + lazy var mockScanQueryData: [BrokerProfileQueryData] = { + let mockNilPreferredRunDateQueryData = Array(1...10).map { _ in + BrokerProfileQueryData.mock(preferredRunDate: nil) + } + let mockPastQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0)) + } + let mockFutureQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0)) + } + + return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData + }() + + func testWhenFilteringOptOutOperationData_thenAllButFuturePreferredRunDateIsReturned() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.count, 30) // all jobs + XCTAssertEqual(operationData2.count, 20) // child broker + past jobs + XCTAssertEqual(operationData3.count, 10) // child broker jobs + XCTAssertEqual(operationData4.count, 30) // all jobs + } + + func testWhenFilteringScanOperationData_thenPreferredRunDatePriorToPriorityDateIsReturned() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.count, 30) // all jobs + XCTAssertEqual(operationData2.count, 10) // past jobs + XCTAssertEqual(operationData3.count, 0) // no jobs + XCTAssertEqual(operationData4.count, 20) // past + future jobs + } + + func testFilteringAllOperationData() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.filter { $0 is ScanJobData }.count, 30) // all jobs + XCTAssertEqual(operationData1.filter { $0 is OptOutJobData }.count, 30) // all jobs + XCTAssertEqual(operationData1.count, 30+30) + + XCTAssertEqual(operationData2.filter { $0 is ScanJobData }.count, 10) // past jobs + XCTAssertEqual(operationData2.filter { $0 is OptOutJobData }.count, 20) // child broker + past jobs + XCTAssertEqual(operationData2.count, 10+20) + + XCTAssertEqual(operationData3.filter { $0 is ScanJobData }.count, 0) // no jobs + XCTAssertEqual(operationData3.filter { $0 is OptOutJobData }.count, 10) // child broker jobs + XCTAssertEqual(operationData3.count, 0+10) + + XCTAssertEqual(operationData4.filter { $0 is ScanJobData }.count, 20) // past + future jobs + XCTAssertEqual(operationData4.filter { $0 is OptOutJobData }.count, 30) // all jobs + XCTAssertEqual(operationData4.count, 20+30) + } +} diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift index 1308534ca3..0fbc1936ed 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift @@ -117,6 +117,13 @@ extension BrokerProfileQueryData { return [broker1Data, broker2Data, broker3Data] } + static func createOptOutJobData(extractedProfileId: Int64, brokerId: Int64, profileQueryId: Int64, preferredRunDate: Date?) -> OptOutJobData { + + let extractedProfile = ExtractedProfile(id: extractedProfileId) + + return OptOutJobData(brokerId: brokerId, profileQueryId: profileQueryId, createdDate: .now, preferredRunDate: preferredRunDate, historyEvents: [], attemptCount: 0, extractedProfile: extractedProfile) + } + static func createOptOutJobData(extractedProfileId: Int64, brokerId: Int64, profileQueryId: Int64, startEventHoursAgo: Int, requestEventHoursAgo: Int, jobCreatedHoursAgo: Int) -> OptOutJobData { let extractedProfile = ExtractedProfile(id: extractedProfileId) From 24a8136cbd76aef2a349a58b9d347e9f56a1a826 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Wed, 27 Nov 2024 01:21:44 -0500 Subject: [PATCH 06/22] Update comments --- .../DataBrokerOperationTests.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift index bde629a234..ad5a8aedb1 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift @@ -23,7 +23,7 @@ final class DataBrokerOperationTests: XCTestCase { lazy var mockOptOutQueryData: [BrokerProfileQueryData] = { let brokerId: Int64 = 1 - let mockChildBrokerQueryData = Array(1...10).map { + let mockNilPreferredRunDateQueryData = Array(1...10).map { BrokerProfileQueryData.mock(preferredRunDate: nil, optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: nil)]) } let mockPastQueryData = Array(1...10).map { @@ -33,7 +33,7 @@ final class DataBrokerOperationTests: XCTestCase { BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowPlus(hours: $0))]) } - return mockChildBrokerQueryData + mockPastQueryData + mockFutureQueryData + return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData }() lazy var mockScanQueryData: [BrokerProfileQueryData] = { @@ -57,8 +57,8 @@ final class DataBrokerOperationTests: XCTestCase { let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantFuture) XCTAssertEqual(operationData1.count, 30) // all jobs - XCTAssertEqual(operationData2.count, 20) // child broker + past jobs - XCTAssertEqual(operationData3.count, 10) // child broker jobs + XCTAssertEqual(operationData2.count, 20) // nil preferred run date + past jobs + XCTAssertEqual(operationData3.count, 10) // nil preferred run date jobs XCTAssertEqual(operationData4.count, 30) // all jobs } @@ -85,11 +85,11 @@ final class DataBrokerOperationTests: XCTestCase { XCTAssertEqual(operationData1.count, 30+30) XCTAssertEqual(operationData2.filter { $0 is ScanJobData }.count, 10) // past jobs - XCTAssertEqual(operationData2.filter { $0 is OptOutJobData }.count, 20) // child broker + past jobs + XCTAssertEqual(operationData2.filter { $0 is OptOutJobData }.count, 20) // nil preferred run date + past jobs XCTAssertEqual(operationData2.count, 10+20) XCTAssertEqual(operationData3.filter { $0 is ScanJobData }.count, 0) // no jobs - XCTAssertEqual(operationData3.filter { $0 is OptOutJobData }.count, 10) // child broker jobs + XCTAssertEqual(operationData3.filter { $0 is OptOutJobData }.count, 10) // nil preferred run date jobs XCTAssertEqual(operationData3.count, 0+10) XCTAssertEqual(operationData4.filter { $0 is ScanJobData }.count, 20) // past + future jobs From 2ffca99967b98eb33028e26f2e74dba8b731ae12 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Wed, 27 Nov 2024 22:21:53 -0500 Subject: [PATCH 07/22] Use distantFuture for preferred run date --- .../OperationPreferredDateCalculator.swift | 2 +- ...kerProfileQueryOperationManagerTests.swift | 35 +++---- ...perationPreferredDateCalculatorTests.swift | 97 +++++++++---------- 3 files changed, 63 insertions(+), 71 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift index dae2ad1b09..acc78af0ac 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift @@ -82,7 +82,7 @@ struct OperationPreferredDateCalculator { case .optOutStarted, .scanStarted, .noMatchFound: return currentPreferredRunDate case .optOutConfirmed, .optOutRequested: - return nil + return Date.distantFuture } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift index 5e772ac0ee..9bf5359ab7 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift @@ -369,24 +369,17 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { } } - func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdatedAndPreferredRunDateIsSetToNil() async { + func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdated() async { do { - let extractedProfileId: Int64 = 1 - let brokerId: Int64 = 1 - let profileQueryId: Int64 = 1 - let mockHistoryEvent = HistoryEvent(extractedProfileId: extractedProfileId, brokerId: brokerId, profileQueryId: profileQueryId, type: .optOutRequested) - let mockBrokerProfileQuery = BrokerProfileQueryData( - dataBroker: .mock, - profileQuery: .mock, - scanJobData: .mock, - optOutJobData: [.mock(with: .mockWithoutRemovedDate, preferredRunDate: Date(), historyEvents: [mockHistoryEvent])] - ) - mockWebOperationRunner.scanResults = [.mockWithoutId] - mockDatabase.brokerProfileQueryDataToReturn = [mockBrokerProfileQuery] _ = try await sut.runScanOperation( on: mockWebOperationRunner, - brokerProfileQueryData: mockBrokerProfileQuery, + brokerProfileQueryData: .init( + dataBroker: .mock, + profileQuery: .mock, + scanJobData: .mock, + optOutJobData: [OptOutJobData.mock(with: .mockWithoutRemovedDate)] + ), database: mockDatabase, notificationCenter: .default, pixelHandler: MockDataBrokerProtectionPixelsHandler(), @@ -396,8 +389,6 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(mockDatabase.optOutEvents.contains(where: { $0.type == .optOutConfirmed })) XCTAssertTrue(mockDatabase.wasUpdateRemoveDateCalled) XCTAssertNotNil(mockDatabase.extractedProfileRemovedDate) - XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut) } catch { XCTFail("Should not throw") } @@ -841,7 +832,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: Date().addingTimeInterval(schedulingConfig.confirmOptOutScan.hoursToSeconds))) } - func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToNil() throws { + func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToDistantFuture() throws { let brokerId: Int64 = 1 let profileQueryId: Int64 = 1 let extractedProfileId: Int64 = 1 @@ -851,7 +842,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: schedulingConfig, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut) + XCTAssertTrue(mockDatabase.lastPreferredRunDateOnOptOut!.isInDistantFuture()) } func testWhenUpdatingDatesAndLastEventIsMatchesFound_thenWeSetScanPreferredDateToMaintanence() throws { @@ -918,7 +909,9 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { // If the date is not going to be set, we don't call the database function XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + + XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + XCTAssertTrue(mockDatabase.lastPreferredRunDateOnOptOut!.isInDistantFuture()) } func testUpdatingScanDateFromScan_thenScanDoesNotRespectMostRecentDate() throws { @@ -942,8 +935,10 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: config, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: expectedPreferredRunDate), "\(String(describing: mockDatabase.lastPreferredRunDateOnScan)) is not equal to \(expectedPreferredRunDate)") + + XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + XCTAssertTrue(mockDatabase.lastPreferredRunDateOnOptOut!.isInDistantFuture()) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index 555c19a9e7..73f3384479 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -498,9 +498,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) } - func testOptOutConfirmedWithCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - + func testOptOutConfirmedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -509,18 +507,16 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) } - func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - + func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -529,18 +525,16 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) } - func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - + func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -549,18 +543,16 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) } func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -569,13 +561,13 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) } func testScanStarted_thenOptOutDoesNotChange() throws { @@ -779,8 +771,6 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutConfirmedWithRecentDate_thenOptOutDateDoesNotChange() throws { - let expectedOptOutDate: Date? = nil - let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -789,16 +779,17 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + + XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) } - func testOptOutRequestedWithRecentDate_thenOptOutDateDoesNotChange() throws { + func testOptOutRequestedWithRecentDate_thenOutOutIsNotScheduled() throws { let expectedOptOutDate: Date? = nil let historyEvents = [ @@ -809,13 +800,13 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) } func testScanStartedWithRecentDate_thenOptOutDateDoesNotChange() throws { @@ -844,3 +835,9 @@ struct MockDate: DateProtocol { return Date(timeIntervalSince1970: 0) } } + +extension Date { + func isInDistantFuture() -> Bool { + self > Date(timeIntervalSinceNow: .days(365_000)) + } +} From 173ffef6233f2258028b0e6f105e18b80e668ff1 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Thu, 28 Nov 2024 11:31:39 -0500 Subject: [PATCH 08/22] Fix SwiftLint --- .../OperationPreferredDateCalculatorTests.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index 73f3384479..0b9a6fb0e2 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -785,7 +785,6 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { schedulingConfig: schedulingConfig, attemptCount: 0) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) } From f629dbf3c0ffa2f22cf9e60a5771887a06d03883 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Thu, 28 Nov 2024 12:11:23 -0500 Subject: [PATCH 09/22] Add tests --- ...perationPreferredDateCalculatorTests.swift | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index 0b9a6fb0e2..ad0925ce9c 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -25,7 +25,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { private let schedulingConfig = DataBrokerScheduleConfig( retryError: 48, confirmOptOutScan: 2000, - maintenanceScan: 3000, + maintenanceScan: 120, maxAttempts: 3 ) @@ -750,6 +750,66 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } } + func testChildBrokerTurnsParentBroker_whenFirstOptOutSucceeds_thenOptOutDateIsSetToDistantFuture() throws { + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .optOutRequested), + ] + let calculator = OperationPreferredDateCalculator() + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) + + XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + } + + func testChildBrokerTurnsParentBroker_whenFirstOptOutFails_thenOptOutIsScheduled() throws { + let expectedOptOutDate = Calendar.current.date(byAdding: .hour, value: 2, to: Date())! + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .error(error: .malformedURL)), + ] + let calculator = OperationPreferredDateCalculator() + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: proposedOptOutDate)) + } + + func testRequestedOptOut_whenProfileReappears_thenOptOutIsScheduled() throws { + let expectedOptOutDate = Date() + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .optOutRequested, + date: .nowMinus(hours: 24*10)), + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .reAppearence), + ] + let calculator = OperationPreferredDateCalculator() + let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: .distantFuture, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: proposedOptOutDate)) + } + func testOptOutStartedWithRecentDate_thenOptOutDateDoesNotChange() throws { let expectedOptOutDate = Date() From e6b3bdde18f74770ee28eee0fa29284ea8550a90 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Mon, 2 Dec 2024 11:33:49 -0500 Subject: [PATCH 10/22] Update test cases --- .../OperationPreferredDateCalculator.swift | 6 ++- ...kerProfileQueryOperationManagerTests.swift | 8 ++-- ...perationPreferredDateCalculatorTests.swift | 44 ++++++++++--------- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift index acc78af0ac..e787f119b7 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift @@ -81,8 +81,10 @@ struct OperationPreferredDateCalculator { return date.now.addingTimeInterval(calculateNextRunDateOnError(schedulingConfig: schedulingConfig, historyEvents: historyEvents)) case .optOutStarted, .scanStarted, .noMatchFound: return currentPreferredRunDate - case .optOutConfirmed, .optOutRequested: - return Date.distantFuture + case .optOutConfirmed: + return nil + case .optOutRequested: + return date.now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift index 9bf5359ab7..869c1e855a 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift @@ -832,7 +832,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: Date().addingTimeInterval(schedulingConfig.confirmOptOutScan.hoursToSeconds))) } - func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToDistantFuture() throws { + func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToMaintenance() throws { let brokerId: Int64 = 1 let profileQueryId: Int64 = 1 let extractedProfileId: Int64 = 1 @@ -842,7 +842,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: schedulingConfig, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertTrue(mockDatabase.lastPreferredRunDateOnOptOut!.isInDistantFuture()) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds))) } func testWhenUpdatingDatesAndLastEventIsMatchesFound_thenWeSetScanPreferredDateToMaintanence() throws { @@ -911,7 +911,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertTrue(mockDatabase.lastPreferredRunDateOnOptOut!.isInDistantFuture()) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.maintenanceScan.hoursToSeconds))) } func testUpdatingScanDateFromScan_thenScanDoesNotRespectMostRecentDate() throws { @@ -938,7 +938,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: expectedPreferredRunDate), "\(String(describing: mockDatabase.lastPreferredRunDateOnScan)) is not equal to \(expectedPreferredRunDate)") XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertTrue(mockDatabase.lastPreferredRunDateOnOptOut!.isInDistantFuture()) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.maintenanceScan.hoursToSeconds))) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index ad0925ce9c..c569cf2eea 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -513,7 +513,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { schedulingConfig: schedulingConfig, attemptCount: 0) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + XCTAssertNil(proposedOptOutDate) } func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { @@ -531,10 +531,12 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { schedulingConfig: schedulingConfig, attemptCount: 0) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + XCTAssertNil(proposedOptOutDate) } func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -547,12 +549,15 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { historyEvents: historyEvents, extractedProfileID: nil, schedulingConfig: schedulingConfig, - attemptCount: 0) + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) } - func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNil() throws { + func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -565,9 +570,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { historyEvents: historyEvents, extractedProfileID: nil, schedulingConfig: schedulingConfig, - attemptCount: 0) + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) } func testScanStarted_thenOptOutDoesNotChange() throws { @@ -750,7 +756,9 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } } - func testChildBrokerTurnsParentBroker_whenFirstOptOutSucceeds_thenOptOutDateIsSetToDistantFuture() throws { + func testChildBrokerTurnsParentBroker_whenFirstOptOutSucceeds_thenOptOutDateIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -762,9 +770,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { historyEvents: historyEvents, extractedProfileID: 1, schedulingConfig: schedulingConfig, - attemptCount: 1) + attemptCount: 1, + date: MockDate()) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) } func testChildBrokerTurnsParentBroker_whenFirstOptOutFails_thenOptOutIsScheduled() throws { @@ -845,11 +854,11 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { schedulingConfig: schedulingConfig, attemptCount: 0) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + XCTAssertNil(proposedOptOutDate) } func testOptOutRequestedWithRecentDate_thenOutOutIsNotScheduled() throws { - let expectedOptOutDate: Date? = nil + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -863,9 +872,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { historyEvents: historyEvents, extractedProfileID: nil, schedulingConfig: schedulingConfig, - attemptCount: 0) + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(proposedOptOutDate!.isInDistantFuture()) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) } func testScanStartedWithRecentDate_thenOptOutDateDoesNotChange() throws { @@ -894,9 +904,3 @@ struct MockDate: DateProtocol { return Date(timeIntervalSince1970: 0) } } - -extension Date { - func isInDistantFuture() -> Bool { - self > Date(timeIntervalSinceNow: .days(365_000)) - } -} From f4f463697f1799f22892327b5b6c6dca4b9e0b6c Mon Sep 17 00:00:00 2001 From: Anh Do Date: Mon, 2 Dec 2024 12:03:13 -0500 Subject: [PATCH 11/22] Add optOutReattempt --- .../DataBrokerProtection/Model/DataBroker.swift | 9 +++++++++ .../DataBrokerProfileQueryOperationManagerTests.swift | 8 ++++---- .../OperationPreferredDateCalculatorTests.swift | 10 +++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift index 269eea17d8..af69e92290 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift @@ -25,6 +25,15 @@ struct DataBrokerScheduleConfig: Codable { let confirmOptOutScan: Int let maintenanceScan: Int let maxAttempts: Int + + var optOutReattempt: Int { + let interval = maintenanceScan + guard interval > confirmOptOutScan else { + assertionFailure("We don't want another opt-out attempt before the next scan") + return Int.max + } + return interval + } } extension Int { diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift index 869c1e855a..8a3fa74edd 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift @@ -832,7 +832,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: Date().addingTimeInterval(schedulingConfig.confirmOptOutScan.hoursToSeconds))) } - func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToMaintenance() throws { + func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToOptOutReattempt() throws { let brokerId: Int64 = 1 let profileQueryId: Int64 = 1 let extractedProfileId: Int64 = 1 @@ -842,7 +842,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: schedulingConfig, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds))) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds))) } func testWhenUpdatingDatesAndLastEventIsMatchesFound_thenWeSetScanPreferredDateToMaintanence() throws { @@ -911,7 +911,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.maintenanceScan.hoursToSeconds))) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.optOutReattempt.hoursToSeconds))) } func testUpdatingScanDateFromScan_thenScanDoesNotRespectMostRecentDate() throws { @@ -938,7 +938,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: expectedPreferredRunDate), "\(String(describing: mockDatabase.lastPreferredRunDateOnScan)) is not equal to \(expectedPreferredRunDate)") XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.maintenanceScan.hoursToSeconds))) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.optOutReattempt.hoursToSeconds))) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index c569cf2eea..2a4759ce52 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -24,7 +24,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { private let schedulingConfig = DataBrokerScheduleConfig( retryError: 48, - confirmOptOutScan: 2000, + confirmOptOutScan: 72, maintenanceScan: 120, maxAttempts: 3 ) @@ -535,7 +535,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -556,7 +556,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -757,7 +757,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testChildBrokerTurnsParentBroker_whenFirstOptOutSucceeds_thenOptOutDateIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -858,7 +858,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutRequestedWithRecentDate_thenOutOutIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, From a6fefeebe8b8a3f8a039777e46de196c930be58a Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 24 Dec 2024 16:17:09 +0700 Subject: [PATCH 12/22] Update test cases --- .../Operations/DataBrokerOperation.swift | 4 +- ...perationPreferredDateCalculatorTests.swift | 124 +++++++++--------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index d24eefb993..b08867d313 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -222,8 +222,8 @@ extension Array where Element == BrokerJobData { /// - Jobs with a preferred run date on or before the priority date are included. /// /// Note: Opt-out jobs without a preferred run date may be: - /// 1. From child brokers (skipped during runOptOutOperation) - /// 2. From former child brokers now acting as parent brokers (processed if extractedProfile hasn't been removed) + /// 1. From child brokers (will be skipped during runOptOutOperation). + /// 2. From former child brokers now acting as parent brokers (will be processed if extractedProfile hasn't been removed). func filtered(using priorityDate: Date) -> [BrokerJobData] { filter { jobData in guard let preferredRunDate = jobData.preferredRunDate else { diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index 2a4759ce52..b602b1f3d8 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -507,13 +507,13 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertNil(proposedOptOutDate) + XCTAssertNil(actualOptOutDate) } func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { @@ -525,13 +525,13 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertNil(proposedOptOutDate) + XCTAssertNil(actualOptOutDate) } func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { @@ -545,14 +545,14 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0, - date: MockDate()) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) } func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { @@ -566,14 +566,14 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0, - date: MockDate()) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) } func testScanStarted_thenOptOutDoesNotChange() throws { @@ -608,10 +608,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) } @@ -766,14 +766,14 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { type: .optOutRequested), ] let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: 1, - schedulingConfig: schedulingConfig, - attemptCount: 1, - date: MockDate()) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) } func testChildBrokerTurnsParentBroker_whenFirstOptOutFails_thenOptOutIsScheduled() throws { @@ -786,13 +786,13 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { type: .error(error: .malformedURL)), ] let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: 1, - schedulingConfig: schedulingConfig, - attemptCount: 1) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: proposedOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) } func testRequestedOptOut_whenProfileReappears_thenOptOutIsScheduled() throws { @@ -810,13 +810,13 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { type: .reAppearence), ] let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: .distantFuture, - historyEvents: historyEvents, - extractedProfileID: 1, - schedulingConfig: schedulingConfig, - attemptCount: 1) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: .distantFuture, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: proposedOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) } func testOptOutStartedWithRecentDate_thenOptOutDateDoesNotChange() throws { @@ -848,13 +848,13 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertNil(proposedOptOutDate) + XCTAssertNil(actualOptOutDate) } func testOptOutRequestedWithRecentDate_thenOutOutIsNotScheduled() throws { @@ -868,14 +868,14 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let proposedOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0, - date: MockDate()) + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: proposedOptOutDate, date2: expectedOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) } func testScanStartedWithRecentDate_thenOptOutDateDoesNotChange() throws { From d776ea0474da361f4b16a7a20198b5840c402326 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 24 Dec 2024 16:36:40 +0700 Subject: [PATCH 13/22] Oops --- .../Operations/OperationPreferredDateCalculator.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift index e787f119b7..10e0b7c9f6 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift @@ -84,7 +84,7 @@ struct OperationPreferredDateCalculator { case .optOutConfirmed: return nil case .optOutRequested: - return date.now.addingTimeInterval(schedulingConfig.maintenanceScan.hoursToSeconds) + return date.now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) } } From 12de5257ebb38d3c66e158b6de810172a97c27a1 Mon Sep 17 00:00:00 2001 From: Brian Hall Date: Thu, 19 Dec 2024 10:49:50 +0100 Subject: [PATCH 14/22] Convert Verecor child sites to be parent sites # Conflicts: # LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json --- .../Resources/JSON/backgroundcheck.run.json | 69 +++++++++++- .../Resources/JSON/clubset.com.json | 79 +++++++++++++- .../Resources/JSON/councilon.com.json | 79 +++++++++++++- .../Resources/JSON/curadvisor.com.json | 79 +++++++++++++- .../Resources/JSON/kwold.com.json | 100 +++++++++++++++--- .../Resources/JSON/newenglandfacts.com.json | 69 +++++++++++- .../JSON/people-background-check.com.json | 69 +++++++++++- .../Resources/JSON/vericora.com.json | 79 +++++++++++++- .../Resources/JSON/veriforia.com.json | 79 +++++++++++++- .../Resources/JSON/virtory.com.json | 79 +++++++++++++- .../Resources/JSON/wellnut.com.json | 79 +++++++++++++- 11 files changed, 808 insertions(+), 52 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json index d8ba2b8237..973b139e4c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json @@ -1,8 +1,7 @@ { "name": "backgroundcheck.run", "url": "backgroundcheck.run", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1677736800000, "optOutUrl": "https://backgroundcheck.run/ng/control/privacy", "steps": [ @@ -51,8 +50,70 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://backgroundcheck.run/ng/control/privacy", + "id": "fa29793c-3f85-4f01-a5fe-4ffcc26c197c" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#name" + }, + { + "type": "email", + "selector": "#email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "71be571c-ce0c-43cb-afad-ae6547d44726" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "e8794659-162d-4de7-9845-bbd140c54a00" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "6ab596e2-3642-4dba-97f0-1270d8feefd1" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "9431f4e4-140a-4ade-8e74-3b7917b6ab2b" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "92e1c4ff-4466-42e7-b2dd-70a319af48da" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json index 3561e4963f..03f73fa09c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json @@ -1,8 +1,7 @@ { "name": "Clubset", "url": "clubset.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1702965600000, "optOutUrl": "https://clubset.com/private/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://clubset.com/private/control/privacy", + "id": "87fb7e0b-87ea-413d-847a-e88b3d023776" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "48c3acd8-61fc-4680-9811-77dc5ba9c9a6" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "47d8cf16-354f-4359-ac37-05c617d5f03e" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "eb5e43df-d4dd-45e8-8192-bfcb3e814ed5" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "2521002e-f825-4a93-aa6e-966f499096d1" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "f72634ca-60c3-450f-9656-52d924d8361f" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "075640a6-f010-4135-9b46-25655e5eadd1" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "96a083a1-f84b-42df-89bd-b5d0601468b2" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json index f15cee8c2e..e5eb48aae1 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json @@ -1,8 +1,7 @@ { "name": "Councilon", "url": "councilon.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1702965600000, "optOutUrl": "https://councilon.com/ex/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://councilon.com/ex/control/privacy", + "id": "9f4ef020-811e-4d02-8622-ebe7a714a0d0" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "8e7adf0a-58fb-4e57-b3b3-79cd99eeb395" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "c80c3e5c-fa66-45c8-816b-bb320a247777" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "30cfa92e-627c-4b00-9fd1-032952338468" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "1d3fe64a-d23d-448c-bf34-aeccf32beb87" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "e6840eae-b334-431d-bbb8-94189e09023d" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "8983034c-211a-4d98-8f31-bdf28a4e5011" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "ffb143e1-a408-456b-9e27-d5b199cd0b52" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json index f415eab9f9..8176c3940e 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json @@ -1,8 +1,7 @@ { "name": "CurAdvisor", "url": "curadvisor.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1703052000000, "optOutUrl": "https://curadvisor.com/nada/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://curadvisor.com/nada/control/privacy", + "id": "0a0fd3f4-4505-4ebc-bbbe-819b86ac18a7" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "0cb160f0-3475-4476-a87f-2a084bcbd4d8" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "5ebf6fc6-a3ee-41f2-875f-82e9d11d24e9" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "22ec8469-c3f8-41e5-8537-6406e817da06" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "d02dd323-1508-4be7-b2d2-72f4bd67fc95" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "be51eb91-bba3-4f2d-8e33-d973372e281d" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "e417378b-bb62-4d63-ad8b-f2e1c3b30e1a" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "e47c20c6-c45f-4d75-b514-06ffa97e35d5" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json index b90a1fae58..57331424cc 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json @@ -1,10 +1,14 @@ { - "name": "Kwold", - "url": "kwold.com", - "version": "0.5.0", - "addedDatetime": 1702965600000, - "optOutUrl": "https://kwold.com/ns/control/privacy", - "steps": [ + "name": "Kwold", + "url": "kwold.com", + "version": "0.5.0", + "addedDatetime": 1702965600000, + "optOutUrl": "https://kwold.com/ns/control/privacy", + "steps": [ + { + "stepType": "scan", + "scanType": "templatedUrl", + "actions": [ { "stepType": "scan", "scanType": "templatedUrl", @@ -137,11 +141,83 @@ } ] } - ], - "schedulingConfig": { - "retryError": 48, - "confirmOptOutScan": 72, - "maintenanceScan": 120, - "maxAttempts": -1 + ] + }, + { + "stepType": "optOut", + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://kwold.com/ns/control/privacy", + "id": "037f7920-b9e7-4214-a937-171ec641d641" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "5b9de12f-a52e-4bd0-b6ac-6884377d309b" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "48e5e7a8-af33-4629-a849-2cf926a518a3" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "bc2d26dc-3eef-478a-a04b-5671a1dbdf8b" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "7f2a685e-ddad-4c5a-8e80-a6d3a690851f" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "3a8a6e9d-c9a0-4e59-a8a4-fe4a05f3ce68" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "93ccf84a-a5ce-4dcf-8a78-143610723488" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" + } + ] } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json index 9be7b4f7d9..ce93b0c82f 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json @@ -1,8 +1,7 @@ { "name": "New England Facts", "url": "newenglandfacts.com", - "version": "0.3.0", - "parent": "verecor.com", + "version": "0.4.0", "addedDatetime": 1703052000000, "optOutUrl": "https://newenglandfacts.com/ng/control/privacy", "steps": [ @@ -61,8 +60,70 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://newenglandfacts.com/ng/control/privacy", + "id": "08332622-1749-4bdc-97d0-2366fef87522" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#name" + }, + { + "type": "email", + "selector": "#email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "e3fdaaf0-70a5-4b7b-afa8-18306276f13a" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "05bcd52d-21c7-4c00-bb18-b1de4b6803aa" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "37511ba6-e622-4614-8657-d4523f650a18" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "80f13572-98f8-4aeb-a77c-25dcc863681a" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "fd232579-b39e-40c8-ab33-596b7e01bf33" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json index cad2226e09..ba41e564c6 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json @@ -1,8 +1,7 @@ { "name": "People Background Check", "url": "people-background-check.com", - "version": "0.5.0", - "parent": "verecor.com", + "version": "0.6.0", "addedDatetime": 1702965600000, "optOutUrl": "https://people-background-check.com/ng/control/privacy", "steps": [ @@ -51,8 +50,70 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://people-background-check.com/ng/control/privacy", + "id": "bc160586-6a98-4eac-9fbf-fa7d24ef8bd7" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#name" + }, + { + "type": "email", + "selector": "#email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "02e4ce59-a0e1-42cb-9adc-d07e4f28a7ff" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "3ab8814b-39c1-4d3b-ab1d-5cbc69e43919" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "b3260029-4cec-43cf-9da2-dc15467e7264" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "c1b7e968-a183-4d91-88da-b8c70313e9b9" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "14e8f388-a24f-4ce8-a3d5-e3e6eda1e8ef" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json index 110c78ffbb..76a017a3b1 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json @@ -1,8 +1,7 @@ { "name": "Vericora", "url": "vericora.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1677736800000, "optOutUrl": "https://vericora.com/ng/control/privacy", "steps": [ @@ -60,8 +59,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://vericora.com/ng/control/privacy", + "id": "db351d7c-3ca0-4af4-9aed-b2f8fba15622" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "c26a1a7c-46c5-4a10-a9d3-ff0660e15e32" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "09599d30-8953-477a-bb70-8042baa20cb5" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "a2af0d54-7170-4181-908b-669bf4dfbcea" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "caf8c297-2e02-4d0b-a9ef-fb9fdb39315d" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "b22b9195-acae-4c23-aeaf-a45e39b7a776" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "e8a95e02-89a0-4c6b-980d-36db2780b54b" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "9adf03c9-444f-450d-992f-adbbdf80a211" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json index 96a6e53137..2dd08c02e8 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json @@ -1,8 +1,7 @@ { "name": "Veriforia", "url": "veriforia.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1677736800000, "optOutUrl": "https://veriforia.com/ng/control/privacy", "steps": [ @@ -60,8 +59,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://veriforia.com/ng/control/privacy", + "id": "943c387c-9cb2-42d6-afd5-a48cf4c2cf3d" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "cf78bb16-b608-4a50-975b-844dad3db1db" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "4c0b8064-1e2b-4598-92a9-e832e73726db" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "c53bc12e-0cf6-4628-b99f-3219cb0f0c1a" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "0aaae970-81d9-4d63-86d0-da4d0a3f5cf4" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "9b6fad42-babc-4777-92af-d415ea9aef5b" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "d0d5f4d0-7183-40f5-b476-4315c751bcc5" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "434d001d-f6cb-4df9-acf3-e62dc821a25a" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json index 02dda9a3df..adc8d3a901 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json @@ -1,8 +1,7 @@ { "name": "Virtory", "url": "virtory.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1703052000000, "optOutUrl": "https://virtory.com/prvt/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://virtory.com/prvt/control/privacy", + "id": "ef272fd4-f284-47ca-aa0c-b4589a85bd81" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "3eb2431f-0c00-440c-8a80-b6d95e0b5ee0" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "2a7550ed-6b80-4d32-adc1-20c96107afcb" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "d7c8d9eb-7239-42d1-9323-566ef1cf4665" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "d4072304-d8c9-47ac-8c47-591a417c15b3" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "c17b21dd-25cf-43ad-9836-01de7f2138cd" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "a3ea0213-bd64-4221-add5-58b517f78223" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "d4d7d0c6-3bba-470a-9fe7-6a154aa07751" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json index 5bcf4aa418..2c6e15a62c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json @@ -1,8 +1,7 @@ { "name": "Wellnut", "url": "wellnut.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1703052000000, "optOutUrl": "https://wellnut.com/noi/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://wellnut.com/noi/control/privacy", + "id": "af96e7b3-fc16-4eed-9307-32da204941a1" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "ce8f2fbd-04f7-48fd-8d67-2cf083b48df3" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "7b01c715-eb45-4b8c-94f0-cb5280a4f1d8" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "d3a79100-785b-4cf8-8bbe-5df701851925" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "d6b36f44-291b-4951-8444-ea4a08a3c1fc" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "a3a7cf9b-4653-4aac-9004-c2f738121ed7" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "cbafc394-018e-47ff-b95e-fbc521508e3f" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "918f6692-c5c4-4072-95c8-795296e1a956" + } + ] } ], "schedulingConfig": { From bef99314fe13aeda66d045b251bca0a1ca4e80ad Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 24 Dec 2024 18:13:41 +0700 Subject: [PATCH 15/22] Add missing property --- .../DataBrokerProtection/Resources/JSON/kwold.com.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json index 57331424cc..05ea56bea3 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json @@ -219,5 +219,11 @@ "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" } ] + }, + "schedulingConfig": { + "retryError": 48, + "confirmOptOutScan": 72, + "maintenanceScan": 120, + "maxAttempts": -1 } } From 80df1907388ea9d2f2b74bed3d3a49b11abb4d22 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 24 Dec 2024 22:35:09 +0700 Subject: [PATCH 16/22] Fix corrupted JSON --- .../Resources/JSON/kwold.com.json | 96 ++----------------- 1 file changed, 7 insertions(+), 89 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json index 05ea56bea3..b90a1fae58 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json @@ -1,14 +1,10 @@ { - "name": "Kwold", - "url": "kwold.com", - "version": "0.5.0", - "addedDatetime": 1702965600000, - "optOutUrl": "https://kwold.com/ns/control/privacy", - "steps": [ - { - "stepType": "scan", - "scanType": "templatedUrl", - "actions": [ + "name": "Kwold", + "url": "kwold.com", + "version": "0.5.0", + "addedDatetime": 1702965600000, + "optOutUrl": "https://kwold.com/ns/control/privacy", + "steps": [ { "stepType": "scan", "scanType": "templatedUrl", @@ -141,85 +137,7 @@ } ] } - ] - }, - { - "stepType": "optOut", - "optOutType": "formOptOut", - "actions": [ - { - "actionType": "navigate", - "url": "https://kwold.com/ns/control/privacy", - "id": "037f7920-b9e7-4214-a937-171ec641d641" - }, - { - "actionType": "fillForm", - "selector": ".ahm", - "elements": [ - { - "type": "fullName", - "selector": "#user_name" - }, - { - "type": "email", - "selector": "#user_email" - }, - { - "type": "profileUrl", - "selector": "#url" - } - ], - "id": "5b9de12f-a52e-4bd0-b6ac-6884377d309b" - }, - { - "actionType": "getCaptchaInfo", - "selector": ".g-recaptcha", - "id": "48e5e7a8-af33-4629-a849-2cf926a518a3" - }, - { - "actionType": "solveCaptcha", - "selector": ".g-recaptcha", - "id": "bc2d26dc-3eef-478a-a04b-5671a1dbdf8b" - }, - { - "actionType": "click", - "elements": [ - { - "type": "button", - "selector": ".//button[@type='submit']" - } - ], - "id": "7f2a685e-ddad-4c5a-8e80-a6d3a690851f" - }, - { - "actionType": "expectation", - "expectations": [ - { - "type": "text", - "selector": "body", - "expect": "Your removal request has been received" - } - ], - "id": "3a8a6e9d-c9a0-4e59-a8a4-fe4a05f3ce68" - }, - { - "actionType": "emailConfirmation", - "pollingTime": 30, - "id": "93ccf84a-a5ce-4dcf-8a78-143610723488" - }, - { - "actionType": "expectation", - "expectations": [ - { - "type": "text", - "selector": "body", - "expect": "Your information control request has been confirmed." - } - ], - "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" - } - ] - }, + ], "schedulingConfig": { "retryError": 48, "confirmOptOutScan": 72, From c00eedf2683dcc2ea91c69d0992ed85657d92970 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Wed, 8 Jan 2025 19:07:02 +0700 Subject: [PATCH 17/22] Revert to old currentScans calculation logic --- .../DataBrokerProtection/UI/UIMapper.swift | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift index 36efa17943..f8befbabcc 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift @@ -32,6 +32,10 @@ struct MapperToUI { return accumulator + brokerQueryData.totalScans } + let currentScans = groupedByBroker.reduce(0) { accumulator, brokerQueryData in + return accumulator + brokerQueryData.currentScans + } + let withSortedGroups = groupedByBroker.map { $0.sortedByLastRunDate() } let sorted = withSortedGroups.sortedByLastRunDate() @@ -40,7 +44,7 @@ struct MapperToUI { brokerQueryGroup.scannedBrokers } - let scanProgress = DBPUIScanProgress(currentScans: partiallyScannedBrokers.count, + let scanProgress = DBPUIScanProgress(currentScans: currentScans, totalScans: totalScans, scannedBrokers: partiallyScannedBrokers) @@ -375,6 +379,18 @@ fileprivate extension Array where Element == BrokerProfileQueryData { return 1 + broker.mirrorSites.filter { $0.shouldWeIncludeMirrorSite() }.count } + var currentScans: Int { + guard let broker = self.first?.dataBroker else { return 0 } + + let didAllQueriesFinished = allSatisfy { $0.scanJobData.lastRunDate != nil } + + if !didAllQueriesFinished { + return 0 + } else { + return 1 + broker.mirrorSites.filter { $0.shouldWeIncludeMirrorSite() }.count + } + } + /// Returns an array of brokers which have been either fully or partially scanned /// /// A broker is considered fully scanned is all scan jobs for that broker have completed. From 09e8f8784f1bfbd6fa9b90ad4d6b0ee8b01edc3a Mon Sep 17 00:00:00 2001 From: Anh Do Date: Thu, 9 Jan 2025 21:22:43 +0700 Subject: [PATCH 18/22] Refactor and update tests --- .../DataBrokerProtection/UI/UIMapper.swift | 27 +++--- .../MapperToUITests.swift | 88 +++++++++++++++---- 2 files changed, 82 insertions(+), 33 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift index 28b8b46488..c144931dab 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift @@ -32,10 +32,6 @@ struct MapperToUI { return accumulator + brokerQueryData.totalScans } - let currentScans = groupedByBroker.reduce(0) { accumulator, brokerQueryData in - return accumulator + brokerQueryData.currentScans - } - let withSortedGroups = groupedByBroker.map { $0.sortedByLastRunDate() } let sorted = withSortedGroups.sortedByLastRunDate() @@ -44,7 +40,7 @@ struct MapperToUI { brokerQueryGroup.scannedBrokers } - let scanProgress = DBPUIScanProgress(currentScans: currentScans, + let scanProgress = DBPUIScanProgress(currentScans: partiallyScannedBrokers.currentScans, totalScans: totalScans, scannedBrokers: partiallyScannedBrokers) @@ -379,18 +375,6 @@ fileprivate extension Array where Element == BrokerProfileQueryData { return 1 + broker.mirrorSites.filter { $0.shouldWeIncludeMirrorSite() }.count } - var currentScans: Int { - guard let broker = self.first?.dataBroker else { return 0 } - - let didAllQueriesFinished = allSatisfy { $0.scanJobData.lastRunDate != nil } - - if !didAllQueriesFinished { - return 0 - } else { - return 1 + broker.mirrorSites.filter { $0.shouldWeIncludeMirrorSite() }.count - } - } - /// Returns an array of brokers which have been either fully or partially scanned /// /// A broker is considered fully scanned is all scan jobs for that broker have completed. @@ -467,6 +451,15 @@ fileprivate extension Array where Element == BrokerProfileQueryData { } } +extension Array where Element == DBPUIScanProgress.ScannedBroker { + /// Number of completed broker scans + var currentScans: Int { + reduce(0) { accumulator, scannedBrokers in + scannedBrokers.status == .completed ? accumulator + 1 : accumulator + } + } +} + fileprivate extension BrokerJobData { var toString: String { if (self as? OptOutJobData) != nil { diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift index edff8837e8..da95b47da0 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift @@ -52,11 +52,16 @@ final class MapperToUITests: XCTestCase { .mock(dataBrokerName: "Broker #2") ] + let expected: [DBPUIScanProgress.ScannedBroker] = [ + .mock("Broker #1", status: .completed), + ] + let result = sut.initialScanState(brokerProfileQueryData) - XCTAssertEqual(result.scanProgress.currentScans, 1) - XCTAssertEqual(result.scanProgress.scannedBrokers.count, result.scanProgress.currentScans) - XCTAssertEqual(result.scanProgress.scannedBrokers.first!.name, "Broker #1") + XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) + XCTAssertEqual(result.scanProgress.scannedBrokers.first!.name, expected.first!.name) XCTAssertTrue(result.resultsFound.isEmpty) } @@ -67,11 +72,16 @@ final class MapperToUITests: XCTestCase { .mock(dataBrokerName: "Broker #2") ] + let expected: [DBPUIScanProgress.ScannedBroker] = [ + .mock("Broker #1", status: .inProgress), + ] + let result = sut.initialScanState(brokerProfileQueryData) - XCTAssertEqual(result.scanProgress.currentScans, 1) - XCTAssertEqual(result.scanProgress.scannedBrokers.count, result.scanProgress.currentScans) - XCTAssertEqual(result.scanProgress.scannedBrokers.first!.name, "Broker #1") + XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) + XCTAssertEqual(result.scanProgress.scannedBrokers.first!.name, expected.first!.name) XCTAssertTrue(result.resultsFound.isEmpty) } @@ -102,12 +112,18 @@ final class MapperToUITests: XCTestCase { .mock(dataBrokerName: "Broker #2", lastRunDate: Date()) ] + let expected: [DBPUIScanProgress.ScannedBroker] = [ + .mock("Broker #1", status: .completed), + .mock("Broker #2", status: .completed), + ] + let result = sut.initialScanState(brokerProfileQueryData) XCTAssertEqual(result.scanProgress.totalScans, result.scanProgress.currentScans) - XCTAssertEqual(result.scanProgress.currentScans, 2) - XCTAssertEqual(result.scanProgress.scannedBrokers.count, result.scanProgress.currentScans) - XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), ["Broker #1", "Broker #2"]) + XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) + XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), expected.map(\.name)) } func testWhenScansHaveDeprecatedProfileQueries_thenThoseAreNotTakenIntoAccount() { @@ -118,12 +134,18 @@ final class MapperToUITests: XCTestCase { .mock(dataBrokerName: "Broker #3", lastRunDate: Date(), extractedProfile: .mockWithRemovedDate, deprecated: true) ] + let expected: [DBPUIScanProgress.ScannedBroker] = [ + .mock("Broker #1", status: .completed), + .mock("Broker #2", status: .completed), + ] + let result = sut.initialScanState(brokerProfileQueryData) XCTAssertEqual(result.scanProgress.totalScans, 2) - XCTAssertEqual(result.scanProgress.currentScans, 2) - XCTAssertEqual(result.scanProgress.scannedBrokers.count, result.scanProgress.currentScans) - XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), ["Broker #1", "Broker #2"]) + XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) + XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), expected.map(\.name)) XCTAssertEqual(result.resultsFound.count, 1) } @@ -136,12 +158,18 @@ final class MapperToUITests: XCTestCase { .mock(dataBrokerName: "Broker #3", lastRunDate: Date(), extractedProfile: .mockWithRemovedDate, deprecated: true) ] + let expected: [DBPUIScanProgress.ScannedBroker] = [ + .mock("Broker #1", status: .completed), + .mock("Broker #2", status: .completed), + ] + let result = sut.initialScanState(brokerProfileQueryData) XCTAssertEqual(result.scanProgress.totalScans, 2) - XCTAssertEqual(result.scanProgress.currentScans, 2) - XCTAssertEqual(result.scanProgress.scannedBrokers.count, result.scanProgress.currentScans) - XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), ["Broker #1", "Broker #2"]) + XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) + XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), expected.map(\.name)) XCTAssertEqual(result.resultsFound.count, 1) } @@ -390,7 +418,7 @@ final class MapperToUITests: XCTestCase { let result = sut.initialScanState(brokerProfileQueryData) - XCTAssertEqual(result.scanProgress.currentScans, 5) + XCTAssertEqual(result.scanProgress.currentScans, 3) XCTAssertEqual(result.scanProgress.scannedBrokers, expected) } @@ -469,6 +497,34 @@ extension DBPUIScanProgress.ScannedBroker { } } +extension Array where Element == BrokerProfileQueryData { + /// Number of completed broker scans, the way it was calculated prior to the introduction of scannedBrokers (1.94.0) + fileprivate var legacyCurrentScans: Int { + filteredProfileQueriesGroupedByBroker.reduce(0) { accumulator, element in + return accumulator + element.value.currentScans + } + } + + private var filteredProfileQueriesGroupedByBroker: [String: [BrokerProfileQueryData]] { + let profileQueriesGroupedByBroker = Dictionary(grouping: self, by: { $0.dataBroker.name }) + return profileQueriesGroupedByBroker.mapValues { queries in + queries.filter { !$0.profileQuery.deprecated } + } + } + + private var currentScans: Int { + guard let broker = self.first?.dataBroker else { return 0 } + + let didAllQueriesFinished = allSatisfy { $0.scanJobData.lastRunDate != nil } + + if !didAllQueriesFinished { + return 0 + } else { + return 1 + broker.mirrorSites.filter { $0.shouldWeIncludeMirrorSite() }.count + } + } +} + extension Date { var yesterday: Date? { From e60fa60d8bf2ceea10acf5a46000e3d0a0723eb0 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Thu, 9 Jan 2025 21:58:13 +0700 Subject: [PATCH 19/22] Add comments --- .../Operations/OperationPreferredDateCalculator.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift index 10e0b7c9f6..ec17b91d39 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift @@ -84,6 +84,10 @@ struct OperationPreferredDateCalculator { case .optOutConfirmed: return nil case .optOutRequested: + // Previously, opt-out jobs with `nil` preferredRunDate were never executed, + // but we need this following the child-to-parent-broker transition + // to prevent repeated scheduling of those former child broker opt-out jobs. + // https://app.asana.com/0/0/1208832818650310/f return date.now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) } } From 1ccfc6dce5d12589ba04bed5ce120215eae2b435 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Mon, 13 Jan 2025 14:27:46 +0700 Subject: [PATCH 20/22] Address PR comments --- .../DataBrokerProtection/Model/DataBroker.swift | 1 + .../Operations/DataBrokerOperation.swift | 4 ++-- .../Sources/DataBrokerProtection/UI/UIMapper.swift | 5 ++--- .../DataBrokerProtectionTests/MapperToUITests.swift | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift index 501934a6ac..aa91c3950e 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift @@ -26,6 +26,7 @@ struct DataBrokerScheduleConfig: Codable { let maintenanceScan: Int let maxAttempts: Int + // Used when scheduling the subsequent opt-out attempt following a successful opt-out request submission var optOutReattempt: Int { let interval = maintenanceScan guard interval > confirmOptOutScan else { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index 8acead829e..7ee7a87300 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -131,7 +131,7 @@ class DataBrokerOperation: Operation, @unchecked Sendable { if let priorityDate = priorityDate { filteredAndSortedOperationsData = operationsData - .filtered(using: priorityDate) + .eligibleForRun(byDate: priorityDate) .sortedByPreferredRunDate() } else { filteredAndSortedOperationsData = operationsData @@ -224,7 +224,7 @@ extension Array where Element == BrokerJobData { /// Note: Opt-out jobs without a preferred run date may be: /// 1. From child brokers (will be skipped during runOptOutOperation). /// 2. From former child brokers now acting as parent brokers (will be processed if extractedProfile hasn't been removed). - func filtered(using priorityDate: Date) -> [BrokerJobData] { + func eligibleForRun(byDate priorityDate: Date) -> [BrokerJobData] { filter { jobData in guard let preferredRunDate = jobData.preferredRunDate else { return jobData is OptOutJobData diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift index c144931dab..fc1fe9d46b 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/UI/UIMapper.swift @@ -40,7 +40,7 @@ struct MapperToUI { brokerQueryGroup.scannedBrokers } - let scanProgress = DBPUIScanProgress(currentScans: partiallyScannedBrokers.currentScans, + let scanProgress = DBPUIScanProgress(currentScans: partiallyScannedBrokers.completeBrokerScansCount, totalScans: totalScans, scannedBrokers: partiallyScannedBrokers) @@ -452,8 +452,7 @@ fileprivate extension Array where Element == BrokerProfileQueryData { } extension Array where Element == DBPUIScanProgress.ScannedBroker { - /// Number of completed broker scans - var currentScans: Int { + var completeBrokerScansCount: Int { reduce(0) { accumulator, scannedBrokers in scannedBrokers.status == .completed ? accumulator + 1 : accumulator } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift index da95b47da0..eb86e913b5 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/MapperToUITests.swift @@ -59,7 +59,7 @@ final class MapperToUITests: XCTestCase { let result = sut.initialScanState(brokerProfileQueryData) XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) - XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.completeBrokerScansCount) XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) XCTAssertEqual(result.scanProgress.scannedBrokers.first!.name, expected.first!.name) XCTAssertTrue(result.resultsFound.isEmpty) @@ -79,7 +79,7 @@ final class MapperToUITests: XCTestCase { let result = sut.initialScanState(brokerProfileQueryData) XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) - XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.completeBrokerScansCount) XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) XCTAssertEqual(result.scanProgress.scannedBrokers.first!.name, expected.first!.name) XCTAssertTrue(result.resultsFound.isEmpty) @@ -121,7 +121,7 @@ final class MapperToUITests: XCTestCase { XCTAssertEqual(result.scanProgress.totalScans, result.scanProgress.currentScans) XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) - XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.completeBrokerScansCount) XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), expected.map(\.name)) } @@ -143,7 +143,7 @@ final class MapperToUITests: XCTestCase { XCTAssertEqual(result.scanProgress.totalScans, 2) XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) - XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.completeBrokerScansCount) XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), expected.map(\.name)) XCTAssertEqual(result.resultsFound.count, 1) @@ -167,7 +167,7 @@ final class MapperToUITests: XCTestCase { XCTAssertEqual(result.scanProgress.totalScans, 2) XCTAssertEqual(result.scanProgress.currentScans, brokerProfileQueryData.legacyCurrentScans) - XCTAssertEqual(result.scanProgress.currentScans, expected.currentScans) + XCTAssertEqual(result.scanProgress.currentScans, expected.completeBrokerScansCount) XCTAssertEqual(result.scanProgress.scannedBrokers.count, expected.count) XCTAssertEqual(result.scanProgress.scannedBrokers.map{ $0.name }.sorted(), expected.map(\.name)) XCTAssertEqual(result.resultsFound.count, 1) From 44350a3d808df2c5beb3f2d7082ec430286c15e6 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 14 Jan 2025 17:05:20 +0700 Subject: [PATCH 21/22] Rename optOutReattempt to hoursUntilNextOptOutAttempt --- .../DataBrokerProtection/Model/DataBroker.swift | 11 ++++------- .../Operations/OperationPreferredDateCalculator.swift | 2 +- .../DataBrokerProfileQueryOperationManagerTests.swift | 6 +++--- .../OperationPreferredDateCalculatorTests.swift | 8 ++++---- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift index aa91c3950e..97bdfe5283 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift @@ -27,13 +27,10 @@ struct DataBrokerScheduleConfig: Codable { let maxAttempts: Int // Used when scheduling the subsequent opt-out attempt following a successful opt-out request submission - var optOutReattempt: Int { - let interval = maintenanceScan - guard interval > confirmOptOutScan else { - assertionFailure("We don't want another opt-out attempt before the next scan") - return Int.max - } - return interval + // This value should be less than `confirmOptOutScan` to ensure the next attempt occurs before + // the confirmation scan. + var hoursUntilNextOptOutAttempt: Int { + maintenanceScan } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift index ec17b91d39..ec8c83b3fb 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift @@ -88,7 +88,7 @@ struct OperationPreferredDateCalculator { // but we need this following the child-to-parent-broker transition // to prevent repeated scheduling of those former child broker opt-out jobs. // https://app.asana.com/0/0/1208832818650310/f - return date.now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) + return date.now.addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift index 8a3fa74edd..ca3760e701 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift @@ -842,7 +842,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: schedulingConfig, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds))) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds))) } func testWhenUpdatingDatesAndLastEventIsMatchesFound_thenWeSetScanPreferredDateToMaintanence() throws { @@ -911,7 +911,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.optOutReattempt.hoursToSeconds))) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.hoursUntilNextOptOutAttempt.hoursToSeconds))) } func testUpdatingScanDateFromScan_thenScanDoesNotRespectMostRecentDate() throws { @@ -938,7 +938,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: expectedPreferredRunDate), "\(String(describing: mockDatabase.lastPreferredRunDateOnScan)) is not equal to \(expectedPreferredRunDate)") XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.optOutReattempt.hoursToSeconds))) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.hoursUntilNextOptOutAttempt.hoursToSeconds))) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index b602b1f3d8..40c691b25f 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -535,7 +535,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -556,7 +556,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -757,7 +757,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testChildBrokerTurnsParentBroker_whenFirstOptOutSucceeds_thenOptOutDateIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -858,7 +858,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutRequestedWithRecentDate_thenOutOutIsNotScheduled() throws { - let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.hoursUntilNextOptOutAttempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, From e19a0aef6b2031fe79fbbdd9e8a515645b4a1e13 Mon Sep 17 00:00:00 2001 From: Anh Do Date: Tue, 14 Jan 2025 20:10:29 +0700 Subject: [PATCH 22/22] Use kwold from main --- .../Resources/JSON/kwold.com.json | 280 +++++++++--------- 1 file changed, 140 insertions(+), 140 deletions(-) diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json index b90a1fae58..4372ba6bb6 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json @@ -1,147 +1,147 @@ { - "name": "Kwold", - "url": "kwold.com", - "version": "0.5.0", - "addedDatetime": 1702965600000, - "optOutUrl": "https://kwold.com/ns/control/privacy", - "steps": [ + "name": "Kwold", + "url": "kwold.com", + "version": "0.5.0", + "addedDatetime": 1702965600000, + "optOutUrl": "https://kwold.com/ns/control/privacy", + "steps": [ + { + "stepType": "scan", + "scanType": "templatedUrl", + "actions": [ { - "stepType": "scan", - "scanType": "templatedUrl", - "actions": [ - { - "actionType": "navigate", - "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", - "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", - "ageRange": [ - "18-30", - "31-40", - "41-50", - "51-60", - "61-70", - "71-80", - "81+" - ] - }, - { - "actionType": "extract", - "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", - "selector": ".card", - "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", - "profile": { - "name": { - "selector": ".card-title", - "beforeText": " ~" - }, - "alternativeNamesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "age": { - "beforeText": "years old", - "selector": ".card-title", - "afterText": " ~" - }, - "addressCityStateList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "relativesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", - "beforeText": ",", - "findElements": true - }, - "profileUrl": { - "selector": "a", - "identifierType": "path", - "identifier": "https://kwold.com/pp/${id}" - } - } - } - ] + "actionType": "navigate", + "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", + "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", + "ageRange": [ + "18-30", + "31-40", + "41-50", + "51-60", + "61-70", + "71-80", + "81+" + ] }, { - "stepType": "optOut", - "optOutType": "formOptOut", - "actions": [ - { - "actionType": "navigate", - "url": "https://kwold.com/ns/control/privacy", - "id": "037f7920-b9e7-4214-a937-171ec641d641" - }, - { - "actionType": "fillForm", - "selector": ".ahm", - "elements": [ - { - "type": "fullName", - "selector": "#user_name" - }, - { - "type": "email", - "selector": "#user_email" - }, - { - "type": "profileUrl", - "selector": "#url" - } - ], - "id": "5b9de12f-a52e-4bd0-b6ac-6884377d309b" - }, - { - "actionType": "getCaptchaInfo", - "selector": ".g-recaptcha", - "id": "48e5e7a8-af33-4629-a849-2cf926a518a3" - }, - { - "actionType": "solveCaptcha", - "selector": ".g-recaptcha", - "id": "bc2d26dc-3eef-478a-a04b-5671a1dbdf8b" - }, - { - "actionType": "click", - "elements": [ - { - "type": "button", - "selector": ".//button[@type='submit']" - } - ], - "id": "7f2a685e-ddad-4c5a-8e80-a6d3a690851f" - }, - { - "actionType": "expectation", - "expectations": [ - { - "type": "text", - "selector": "body", - "expect": "Your removal request has been received" - } - ], - "id": "3a8a6e9d-c9a0-4e59-a8a4-fe4a05f3ce68" - }, - { - "actionType": "emailConfirmation", - "pollingTime": 30, - "id": "93ccf84a-a5ce-4dcf-8a78-143610723488" - }, - { - "actionType": "expectation", - "expectations": [ - { - "type": "text", - "selector": "body", - "expect": "Your information control request has been confirmed." - } - ], - "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" - } - ] + "actionType": "extract", + "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", + "selector": ".card", + "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", + "profile": { + "name": { + "selector": ".card-title", + "beforeText": " ~" + }, + "alternativeNamesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "age": { + "beforeText": "years old", + "selector": ".card-title", + "afterText": " ~" + }, + "addressCityStateList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "relativesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", + "beforeText": ",", + "findElements": true + }, + "profileUrl": { + "selector": "a", + "identifierType": "path", + "identifier": "https://kwold.com/pp/${id}" + } + } } - ], - "schedulingConfig": { - "retryError": 48, - "confirmOptOutScan": 72, - "maintenanceScan": 120, - "maxAttempts": -1 + ] + }, + { + "stepType": "optOut", + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://kwold.com/ns/control/privacy", + "id": "037f7920-b9e7-4214-a937-171ec641d641" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "5b9de12f-a52e-4bd0-b6ac-6884377d309b" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "48e5e7a8-af33-4629-a849-2cf926a518a3" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "bc2d26dc-3eef-478a-a04b-5671a1dbdf8b" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "7f2a685e-ddad-4c5a-8e80-a6d3a690851f" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "3a8a6e9d-c9a0-4e59-a8a4-fe4a05f3ce68" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "93ccf84a-a5ce-4dcf-8a78-143610723488" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" + } + ] } + ], + "schedulingConfig": { + "retryError": 48, + "confirmOptOutScan": 72, + "maintenanceScan": 120, + "maxAttempts": -1 + } }