Delete invalid or non existing objects

configurations/sharepoint-woo/mappings/xxllnc-suite-to-publications.json

@@ -5,7 +5,7 @@
         "title": "omschrijving",
         "summary": "zaaktypeomschrijving",
         "description": "zaaktypeomschrijving",
-        "category": "{% if zaaktypecode|default %}{% set wooVerzoekenEnBesluiten = ['LP00000431', 'B1873', 'cherry'] %}{% set klachtoordelen = ['LP00000091', 'LP00001132', 'LP00000121', 'B0757', 'LP00000832', 'LP00001096'] %}{% if zaaktypecode in wooVerzoekenEnBesluiten %}{{ 'Woo-verzoeken en -besluiten' }}{% elseif zaaktypecode in klachtoordelen %}{{ 'Klachtoordelen' }}{% endif %}{% endif %}",
+        "category": "{% if zaaktypecode|default %}{% set wooVerzoekenEnBesluiten = ['LP00000431', 'B1873'] %}{% set klachtoordelen = ['LP00000091', 'LP00001132', 'LP00000121', 'B0757', 'LP00000832', 'LP00001096'] %}{% if zaaktypecode in wooVerzoekenEnBesluiten %}{{ 'Woo-verzoeken en -besluiten' }}{% elseif zaaktypecode in klachtoordelen %}{{ 'Klachtoordelen' }}{% endif %}{% endif %}",
         "published": "startdatum",
         "modified": "{{ 'now'|date('H:i:sTm-d-Y') }}",
         "attachments": "[{% if files|default %}{% for file in files %} { {% if file['titel']|default %}\"title\": \"{{ file['titel'] }}\",{% endif %}\"labels\": [\"{{ 'Informatieverzoek' }}\"],{% if file['formaat']|default %}\"extension\": \"{{ file['formaat']|split('/')|last }}\",\"type\": \"{{ file['formaat'] }}\",{% endif %}{% if file['inhoud']|default and file['formaat']|default %}\"accessUrl\": \"data:{{ file['formaat'] }};base64,{{ file.inhoud }}\"{% endif %} }{{ loop.last ? '' : ',' }} {% endfor %}{% endif %}]",

configurations/sharepoint-woo/synchronizations/xxllnc-suite-to-publications.json

@@ -5,6 +5,7 @@
 	"sourceId": "1",
 	"sourceType": "api",
 	"sourceHash": "",
+	"sourceHashMapping": "1",
 	"sourceTargetMapping": "1",
 	"sourceConfig": {
         "idPosition": "identificatie",
@@ -13,6 +14,7 @@
         "headers": [],
         "query.startdatum__gte": "2024-08-01",
         "query.einddatum__lt": "2025-01-01",
+        "query.maximaleVertrouwelijkheidaanduiding": "openbaar",
         "usesPagination": "false",
         "extraDataConfigs.0.staticEndpoint": "/tlb/zaaksysteem/api/v1/zaken/{{ originId }}/informatieobjecten",
         "extraDataConfigs.0.mergeExtraData": "true",
@@ -21,5 +23,17 @@
         "extraDataConfigs.0.extraDataConfigPerResult.staticEndpoint": "/tlb/zaaksysteem/api/v1/informatieobjecten/{{ originId }}"
     },
 	"targetId": "1/1",
-	"targetType": "register/schema"
+	"targetType": "register/schema",
+    "conditions": {
+        "if": [
+            {
+            "or": [
+                { "in": [{ "var": "zaaktypecode" }, ["LP00000431", "B1873"]] },
+                { "in": [{ "var": "zaaktypecode" }, ["LP00000091", "LP00001132", "LP00000121", "B0757", "LP00000832", "LP00001096"]] }
+            ]
+            },
+            true,
+            false
+        ]
+    }
 }

lib/Db/SynchronizationContractMapper.php

@@ -117,6 +117,32 @@ public function findOnTarget(string $synchronization, string $targetId): Synchro
         }
     }
 
+    /**
+     * Find all target IDs of synchronization contracts by synchronization ID
+     *
+     * @param string $synchronization The synchronization ID
+     *
+     * @return array An array of target IDs or an empty array if none found
+     */
+    public function findAllBySynchronization(string $synchronizationId): array
+    {
+        // Create query builder
+        $qb = $this->db->getQueryBuilder();
+
+        // Build select query with synchronization ID filter
+        $qb->select('*')
+            ->from('openconnector_synchronization_contracts')
+            ->where(
+                $qb->expr()->eq('synchronization_id', $qb->createNamedParameter($synchronizationId))
+            );
+
+        try {
+            return $this->findEntities($qb);
+        } catch (\Exception $e) {
+            return [];
+        }
+    }
+
     /**
      * Find all synchronization contracts with optional filtering and pagination
      *
@@ -201,6 +227,34 @@ public function updateFromArray(int $id, array $object): SynchronizationContract
         return $this->update($obj);
     }
 
+    /**
+     * Find a synchronization contract by origin ID.
+     *
+     * @param string $originId The origin ID to search for.
+     *
+     * @return SynchronizationContract|null The matching contract or null if not found.
+     */
+    public function findByOriginId(string $originId): ?SynchronizationContract
+    {
+        // Create query builder
+        $qb = $this->db->getQueryBuilder();
+
+        // Build query to find contract matching origin_id
+        $qb->select('*')
+            ->from('openconnector_synchronization_contracts')
+            ->where(
+                $qb->expr()->eq('origin_id', $qb->createNamedParameter($originId))
+            )
+            ->setMaxResults(1); // Ensure only one result is returned
+
+        try {
+            return $this->findEntity($qb); // Use findEntity to return a single result
+        } catch (\OCP\AppFramework\Db\DoesNotExistException $e) {
+            return null; // Return null if no match is found
+        }
+    }
+
+
     /**
      * Find synchronization contracts by type and ID
      *

lib/Service/SynchronizationService.php

@@ -97,7 +97,18 @@ public function synchronize(Synchronization $synchronization, ?bool $isTest = fa
 
         $objectList = $this->getAllObjectsFromSource(synchronization: $synchronization, isTest: $isTest);
 
+        $synchronizedTargetIds = [];
         foreach ($objectList as $key => $object) {
+
+            // Check if object adheres to conditions.
+            // Take note, JsonLogic::apply() returns a range of return types, so checking it with '=== false' or '!== true' does not work properly.
+            if ($synchronization->getConditions() !== [] && !JsonLogic::apply($synchronization->getConditions(), $object)) {
+
+                // @todo log that this object is not valid
+                unset($objectList[$key]);
+                continue;
+            }
+
             // If the source configuration contains a dot notation for the id position, we need to extract the id from the source object
             $originId = $this->getOriginId($synchronization, $object);
 
@@ -138,9 +149,15 @@ public function synchronize(Synchronization $synchronization, ?bool $isTest = fa
                 }
             }
 
-            $this->synchronizationContractMapper->update($synchronizationContract);
+            $synchronizationContract = $this->synchronizationContractMapper->update($synchronizationContract);
+
+            $synchronizedTargetIds[] = $synchronizationContract->getTargetId();
         }
 
+        $this->deleteInvalidObjects(synchronization: $synchronization, synchronizedTargetIds: $synchronizedTargetIds);
+        // @todo log deleted objects count
+
+
 		foreach ($synchronization->getFollowUps() as $followUp) {
 			$followUpSynchronization = $this->synchronizationMapper->find($followUp);
 			$this->synchronize($followUpSynchronization, $isTest);
@@ -294,14 +311,64 @@ private function fetchExtraDataForObject(Synchronization $synchronization, array
         return $extraData;
     }
 
+    /**
+     * Deletes invalid objects associated with a synchronization.
+     *
+     * This function identifies and removes objects that are no longer valid or do not exist
+     * in the source data for a given synchronization. It compares the target IDs from the
+     * synchronization contract with the synchronized target IDs and deletes the unmatched ones.
+     *
+     * @param Synchronization $synchronization       The synchronization entity to process.
+     * @param array           $synchronizedTargetIds An array of target IDs that are still valid in the source.
+     *
+     * @return int The count of objects that were deleted.
+     *
+     * @throws Exception If any database or object deletion errors occur during execution.
+     */
+    public function deleteInvalidObjects(Synchronization $synchronization, array $synchronizedTargetIds): int
+    {
+        $deletedObjectsCount = 0;
+        $type = $synchronization->getTargetType();
+
+        switch ($type) {
+            case 'register/schema':
+
+                $targetIdsToDelete = [];
+                $allContracts = $this->synchronizationContractMapper->findAllBySynchronization($synchronization->getId());
+                $allContractTargetIds = [];
+                foreach ($allContracts as $contract) {
+                    if ($contract->getTargetId() !== null) {
+                        $allContractTargetIds[] = $contract->getTargetId();
+                    }
+                }
+
+                // Check if we have contracts that became invalid or do not exist in the source anymore
+                $targetIdsToDelete = array_diff($allContractTargetIds, $synchronizedTargetIds);
+
+                foreach ($targetIdsToDelete as $targetIdToDelete) {
+                    try {
+                        $synchronizationContract = $this->synchronizationContractMapper->findOnTarget(synchronization: $synchronization->getId(), targetId: $targetIdToDelete);
+                        $synchronizationContract = $this->updateTarget(synchronizationContract: $synchronizationContract, targetObject: [], action: 'delete');
+                        $this->synchronizationContractMapper->update($synchronizationContract);
+                        $deletedObjectsCount++;
+                    } catch (DoesNotExistException $exception) {
+                        // @todo log
+                    }
+                }
+            break;
+        }
+
+        return $deletedObjectsCount;
+
+    }
 
 	/**
 	 * Synchronize a contract
 	 *
 	 * @param SynchronizationContract $synchronizationContract
-	 * @param Synchronization|null $synchronization
-	 * @param array $object
-	 * @param bool|null $isTest False by default, currently added for synchronization-test endpoint
+	 * @param Synchronization|null    $synchronization
+	 * @param array                   $object
+	 * @param bool|null               $isTest                  False by default, currently added for synchronization-test endpoint
 	 *
 	 * @return SynchronizationContract|Exception|array
 	 * @throws ContainerExceptionInterface
@@ -334,18 +401,12 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont
             // The object has not changed and the config has not been updated since last check
 			return $synchronizationContract;
         }
-		
+
         // The object has changed, oke let do mappig and bla die bla
         $synchronizationContract->setOriginHash($originHash);
         $synchronizationContract->setSourceLastChanged(new DateTime());
 		$synchronizationContract->setSourceLastChecked(new DateTime());
 
-		// Check if object adheres to conditions.
-		// Take note, JsonLogic::apply() returns a range of return types, so checking it with '=== false' or '!== true' does not work properly.
-		if ($synchronization->getConditions() !== [] && !JsonLogic::apply($synchronization->getConditions(), $object)) {
-			return $synchronizationContract;
-		}
-
         // If no source target mapping is defined, use original object
         if (empty($synchronization->getSourceTargetMapping()) === true) {
             $targetObject = $object;
@@ -402,17 +463,65 @@ public function synchronizeContract(SynchronizationContract $synchronizationCont
 
     }
 
+    /**
+     * Updates or deletes a target object in the Open Register system.
+     *
+     * This method updates a target object associated with a synchronization contract
+     * or deletes it based on the specified action. It extracts the register and schema
+     * from the target ID and performs the corresponding operation using the object service.
+     *
+     * @param SynchronizationContract $synchronizationContract The synchronization contract being updated.
+     * @param Synchronization         $synchronization         The synchronization entity containing the target ID.
+     * @param array|null              $targetObject            An optional array containing the data for the target object. Defaults to an empty array.
+     * @param string|null             $action                  The action to perform: 'save' (default) to update or 'delete' to remove the target object.
+     *
+     * @return SynchronizationContract The updated synchronization contract with the modified target ID.
+     *
+     * @throws \Exception If an error occurs while interacting with the object service or processing the data.
+     */
+    private function updateTargetOpenRegister(SynchronizationContract $synchronizationContract, Synchronization $synchronization, ?array $targetObject = [], ?string $action = 'save'): SynchronizationContract
+    {
+        // Setup the object service
+        $objectService = $this->containerInterface->get('OCA\OpenRegister\Service\ObjectService');
+
+        // if we already have an id, we need to get the object and update it
+        if ($synchronizationContract->getTargetId() !== null) {
+            $targetObject['id'] = $synchronizationContract->getTargetId();
+        }
+
+        // Extract register and schema from the targetId
+        // The targetId needs to be filled in as: {registerId} + / + {schemaId} for example: 1/1
+        $targetId = $synchronization->getTargetId();
+        list($register, $schema) = explode('/', $targetId);
+
+        // Save the object to the target
+        switch ($action) {
+            case 'save':
+                $target = $objectService->saveObject($register, $schema, $targetObject);
+                // Get the id form the target object
+                $synchronizationContract->setTargetId($target->getUuid());
+                break;
+            case 'delete':
+                $objectService->deleteObject(register: $register, schema: $schema, uuid: $synchronizationContract->getTargetId());
+                $synchronizationContract->setTargetId(null);
+                break;
+        }
+
+        return $synchronizationContract;
+    }
+
 	/**
 	 * Write the data to the target
 	 *
 	 * @param SynchronizationContract $synchronizationContract
-	 * @param array $targetObject
+	 * @param array                   $targetObject
+	 * @param string|null             $action                  Determines what needs to be done with the target object, defaults to 'save'
 	 *
 	 * @return SynchronizationContract
 	 * @throws ContainerExceptionInterface
 	 * @throws NotFoundExceptionInterface
 	 */
-    public function updateTarget(SynchronizationContract $synchronizationContract, array $targetObject): SynchronizationContract
+    public function updateTarget(SynchronizationContract $synchronizationContract, ?array $targetObject = [], ?string $action = 'save'): SynchronizationContract
 	{
          // The function can be called solo set let's make sure we have the full synchronization object
         if (isset($synchronization) === false) {
@@ -429,24 +538,7 @@ public function updateTarget(SynchronizationContract $synchronizationContract, a
 
         switch ($type) {
             case 'register/schema':
-                // Setup the object service
-                $objectService = $this->containerInterface->get('OCA\OpenRegister\Service\ObjectService');
-
-                // if we already have an id, we need to get the object and update it
-                if ($synchronizationContract->getTargetId() !== null) {
-                    $targetObject['id'] = $synchronizationContract->getTargetId();
-                }
-
-                // Extract register and schema from the targetId
-                // The targetId needs to be filled in as: {registerId} + / + {schemaId} for example: 1/1
-                $targetId = $synchronization->getTargetId();
-                list($register, $schema) = explode('/', $targetId);
-
-                // Save the object to the target
-                $target = $objectService->saveObject($register, $schema, $targetObject);
-
-                // Get the id form the target object
-                $synchronizationContract->setTargetId($target->getUuid());
+                $synchronizationContract = $this->updateTargetOpenRegister(synchronizationContract: $synchronizationContract, synchronization: $synchronization, targetObject: $targetObject, action: $action);
                 break;
             case 'api':
                 //@todo: implement

src/entities/synchronization/synchronization.mock.ts

@@ -6,6 +6,7 @@ export const mockSynchronizationData = (): TSynchronization[] => [
 		id: 1,
 		name: 'Synchronization 1',
 		description: 'Synchronization 1',
+		conditions: '{"!!": { "var": "valid" }}',
 		sourceId: 'source1',
 		sourceType: 'api',
 		sourceHash: 'source1',
@@ -29,6 +30,7 @@ export const mockSynchronizationData = (): TSynchronization[] => [
 		id: 2,
 		name: 'Synchronization 2',
 		description: 'Synchronization 2',
+		conditions: '{"!!": { "var": "valid" }}',
 		sourceId: 'source2',
 		sourceType: 'api',
 		sourceHash: 'source2',

src/entities/synchronization/synchronization.ts

@@ -8,6 +8,7 @@ export class Synchronization extends ReadonlyBaseClass implements TSynchronizati
 	public id: number
 	public name: string
 	public description: string
+	public conditions: string
 	public sourceId: string
 	public sourceType: string
 	public sourceHash: string
@@ -32,6 +33,7 @@ export class Synchronization extends ReadonlyBaseClass implements TSynchronizati
 			id: synchronization.id || null,
 			name: synchronization.name || '',
 			description: synchronization.description || '',
+			conditions: synchronization.conditions || '',
 			sourceId: synchronization.sourceId || '',
 			sourceType: synchronization.sourceType || '',
 			sourceHash: synchronization.sourceHash || '',
@@ -60,6 +62,7 @@ export class Synchronization extends ReadonlyBaseClass implements TSynchronizati
 			id: z.number().nullable(),
 			name: z.string(),
 			description: z.string(),
+			conditions: z.string(),
 			sourceId: z.string(),
 			sourceType: z.string(),
 			sourceHash: z.string(),

src/entities/synchronization/synchronization.types.ts

@@ -2,6 +2,7 @@ export type TSynchronization = {
     id: number
     name: string
     description: string
+    conditions: string
     sourceId: string
     sourceType: string
     sourceHash: string