Merge pull request #168 from ConductionNL/feature/CONNECTOR-52/cloud-…

…events

Rules logic and cloud events standard

appinfo/routes.php

@@ -8,6 +8,8 @@
 		'Jobs' => ['url' => 'api/jobs'],
 		'Synchronizations' => ['url' => 'api/synchronizations'],
 		'Consumers' => ['url' => 'api/consumers'],
+		'Rules' => ['url' => 'api/rules'],
+		'Events' => ['url' => 'api/events'],
 	],
 	'routes' => [
 		['name' => 'dashboard#page', 'url' => '/', 'verb' => 'GET'],
@@ -17,23 +19,40 @@
 		['name' => 'jobs#logs', 'url' => '/api/jobs-logs/{id}', 'verb' => 'GET'],
 		['name' => 'endpoints#test', 'url' => '/api/endpoints-test/{id}', 'verb' => 'POST'],
 		['name' => 'endpoints#logs', 'url' => '/api/endpoints-logs/{id}', 'verb' => 'GET'],
+
+		// Synchronization endpoints
 		['name' => 'synchronizations#contracts', 'url' => '/api/synchronizations-contracts/{id}', 'verb' => 'GET'],
 		['name' => 'synchronizations#logs', 'url' => '/api/synchronizations-logs/{id}', 'verb' => 'GET'],
 		['name' => 'synchronizations#test', 'url' => '/api/synchronizations-test/{id}', 'verb' => 'POST'],
 		['name' => 'synchronizations#run', 'url' => '/api/synchronizations-run/{id}', 'verb' => 'POST'],
+
 		// Mapping endpoints
 		['name' => 'mappings#test', 'url' => '/api/mappings/test', 'verb' => 'POST'],
 		['name' => 'mappings#saveObject', 'url' => '/api/mappings/objects', 'verb' => 'POST'],
 		['name' => 'mappings#getObjects', 'url' => '/api/mappings/objects', 'verb' => 'GET'],
+
 		// Running endpoints - allow any path after /api/endpoints/
 		['name' => 'endpoints#handlePath', 'postfix' => 'read', 'url' => '/api/endpoint/{_path}', 'verb' => 'GET', 'requirements' => ['_path' => '.+']],
 		['name' => 'endpoints#handlePath', 'postfix' => 'update', 'url' => '/api/endpoint/{_path}', 'verb' => 'PUT', 'requirements' => ['_path' => '.+']],
 		['name' => 'endpoints#handlePath', 'postfix' => 'partialupdate', 'url' => '/api/endpoint/{_path}', 'verb' => 'PATCH', 'requirements' => ['_path' => '.+']],
 		['name' => 'endpoints#handlePath', 'postfix' => 'create', 'url' => '/api/endpoint/{_path}', 'verb' => 'POST', 'requirements' => ['_path' => '.+']],
 		['name' => 'endpoints#handlePath', 'postfix' => 'destroy', 'url' => '/api/endpoint/{_path}', 'verb' => 'DELETE', 'requirements' => ['_path' => '.+']],
+
 		// Import & Export
 		['name' => 'import#import', 'url' => '/api/import', 'verb' => 'POST'],
-//		['name' => 'import#importUpdate', 'url' => '/api/import/{id}', 'verb' => 'PUT'],
 		['name' => 'export#export', 'url' => '/api/export/{type}/{id}', 'verb' => 'GET'],
+
+		// Event messages
+		['name' => 'events#messages', 'url' => '/api/events/{id}/messages', 'verb' => 'GET'],
+
+		// Subscription management
+		['name' => 'events#subscriptions', 'url' => '/api/events/subscriptions', 'verb' => 'GET'],
+		['name' => 'events#subscriptionMessages', 'url' => '/api/events/subscriptions/{subscriptionId}/messages', 'verb' => 'GET'],
+		['name' => 'events#subscribe', 'url' => '/api/events/subscriptions', 'verb' => 'POST'],
+		['name' => 'events#updateSubscription', 'url' => '/api/events/subscriptions/{subscriptionId}', 'verb' => 'PUT'],
+		['name' => 'events#unsubscribe', 'url' => '/api/events/subscriptions/{subscriptionId}', 'verb' => 'DELETE'],
+
+		// Pull-based delivery
+		['name' => 'events#pull', 'url' => '/api/events/subscriptions/{subscriptionId}/pull', 'verb' => 'GET'],
 	],
 ];

docs/Synchronization/Synchronization.md

@@ -0,0 +1,77 @@
+# Synchronization
+
+Synchronization is a core feature that enables data transfer between different systems and APIs. Here's how it works:
+
+## Overview
+- Synchronizations define how data should be synchronized between a source and target system
+- Each synchronization has a source configuration (where to get data from) and target configuration (where to send data to)
+- The synchronization process handles pagination, data mapping, and maintaining sync state
+
+## Key Components
+
+### Source Configuration
+- Defines where to fetch data from (API endpoint, database etc)
+- Specifies how to map source data to target format (incomming data)
+- Specifies how to locate objects in the response using resultsPosition:
+  - `_root`: Uses the entire response body as the objects array
+  - Dot notation (e.g. `data.items`): Extracts objects from a nested path
+  - Common keys (`items`, `result`, `results`): Automatically checks these standard locations
+  - Custom path: Specify any JSON path to locate the objects
+- Configures pagination settings (paginationQuery)
+- Can include conditions to filter which objects to sync using JSON Logic
+- Source type can be set to API or other supported types
+- Source ID mapping allows specifying position of IDs in source objects
+- Optional endpoint configuration for fetching data
+
+### Target Configuration 
+- Defines where to send synchronized data
+- Specifies how to map target data to source format (outgoing data)
+- Handles create/update/delete operations
+- Target type can be Register/Schema or other supported types
+- Target ID and schema selection for Register/Schema targets
+- Target source mapping for data transformations
+
+### Synchronization Contracts
+- Tracks the sync state for each individual object
+- Stores origin ID and target ID mappings
+- Maintains hashes to detect changes
+- Logs synchronization events and errors
+
+## Process Flow
+1. Fetch objects from source system with pagination
+2. Filter objects based on configured conditions
+3. Create/update synchronization contracts for each object
+4. Transform data according to mapping rules
+5. Write objects to target system (POST/PUT/DELETE)
+6. Update contract status and hashes
+7. Handle any follow-up synchronizations
+
+## Error Handling
+- Rate limiting detection and backoff
+- Logging of failed operations
+- Contract state tracking for retry attempts
+
+The synchronization system provides a robust way to keep data in sync across different systems while maintaining state and handling errors gracefully.
+
+## Form Configuration
+The synchronization form allows configuring:
+
+- Name: Descriptive name for the synchronization
+- Description: Optional details about the synchronization
+- Conditions: JSON Logic conditions for filtering objects
+- Source Configuration:
+  - Source Type: API or other supported types
+  - Source ID: Selection of configured source
+  - Source hash mapping: Hash configuration
+  - Source target mapping: Data mapping rules
+  - Position of ID in source object (optional)
+  - Position of results in source object (optional)
+  - Custom endpoint for data fetching (optional)
+- Target Configuration:
+  - Target Type: Register/Schema or other types
+  - Target ID: Selection of target system
+  - Register and Schema selection for Register/Schema targets
+  - Target source mapping: Data transformation rules
+- Test sync option to validate configuration
+
+![alt text](image.png)

docs/cloudevents.md

@@ -2,12 +2,44 @@
 
 We facilitate subscriptions on a pub/sub model. This is based on [CloudEvents](https://cloudevents.io/) but also supports the [NL GOV profile for CloudEvents](https://www.logius.nl/domeinen/gegevensuitwisseling/nl-gov-profile-cloudevents). More documentation can be found [here](https://gitdocumentatie.logius.nl/publicatie/notificatieservices/CloudEvents-NL/).
 
-## Subsribers 
+# Event System
 
-## Publishers
+## Event Subscriptions
 
-## Endpoints
+The OpenConnector platform implements the [CloudEvents Subscription API specification](https://github.com/cloudevents/spec/blob/main/subscriptions/spec.md) to manage event subscriptions. This allows consumers to subscribe to specific events and receive them through various delivery mechanisms.
 
-## Mappings
+### Subscription Styles
 
-## Events
+The platform supports two subscription styles:
+
+- **Push**: Events are actively sent to the subscriber's endpoint (sink)
+- **Pull**: Subscribers fetch events from the platform
+
+### Subscription Properties
+
+Each subscription contains the following properties:
+
+- `id`: Unique identifier for the subscription
+- `source`: URI identifying where events originate
+- `types`: Array of CloudEvent type values to subscribe to
+- `config`: Subscription-specific configuration
+- `filters`: Array of filter expressions for event matching
+- `sink`: URI where events should be delivered
+- `protocol`: Delivery protocol (HTTP, MQTT, AMQP, etc.)
+- `protocolSettings`: Protocol-specific settings
+- `style`: Delivery style ('push' or 'pull')
+- `status`: Subscription status
+- `userId`: Owner of the subscription
+
+### Filter Dialects
+
+The platform supports the following filter dialects as defined in the CloudEvents specification:
+
+- `exact`: Exact matching of attribute values
+- `prefix`: Prefix matching of attribute values
+- `suffix`: Suffix matching of attribute values
+- `all`: Logical AND of multiple filters
+- `any`: Logical OR of multiple filters
+- `not`: Logical NOT of a filter
+
+### Example Subscription

docs/rules/rules.md

@@ -0,0 +1,100 @@
+# Rules
+
+Rules are components that can be associated with endpoints to add additional functionality. Based on the current codebase, rules are stored as string IDs and can be attached to multiple endpoints.
+
+## Adding Rules to Endpoints
+
+Rules can be added to endpoints through the AddEndpointRule modal component. The process works as follows:
+
+1. Select a rule from the available rule options dropdown
+2. The rule ID is added to the endpoint's `rules` array property
+3. The endpoint is updated with the new rule association
+4. On success, the modal closes automatically after 2 seconds
+
+## Rule Properties
+
+Rules have several key properties that define their behavior:
+
+- uuid: A unique identifier string for the rule
+- name: The display name of the rule
+- description: A detailed description of the rule's purpose
+- action: Specifies when the rule triggers (create, read, update, delete)
+- timing: Controls if rule runs 'before' or 'after' the action (defaults to 'before')
+- conditions: JSON Logic format conditions that determine when rule applies
+- type: The rule type (mapping, error, script, synchronization, authentication, download, upload, locking)
+- configuration: Type-specific configuration stored as JSON
+- order: Integer determining execution order when multiple rules exist
+
+The properties work together to define:
+
+1. When the rule executes (action + timing)
+2. Under what conditions it runs (conditions)
+3. What functionality it provides (type + configuration) 
+4. The sequence of execution (order)
+
+Rules are stored in a structured format but referenced by endpoints using their UUID strings in an array format.
+
+## Rule Implementation
+
+The current implementation shows:
+
+- Rules are managed through a dedicated API endpoint at `/api/rules`
+- Rules can be retrieved and managed through the Rules resource controller
+- Rule IDs are stored and validated as strings within endpoints
+- The endpoint entity ensures rules are always stored as an array
+
+## Rule Types
+
+### Authentication Rules
+
+Authentication rules control access to endpoints by validating user credentials and permissions. Configuration options include:
+
+- type: The authentication method to use
+  - basic: Basic HTTP authentication
+  - jwt: JSON Web Token authentication
+  - jwt-zgw: ZGW-specific JWT authentication
+  - oauth: OAuth 2.0 authentication
+- users: Array of specific users allowed to access the endpoint
+- groups: Array of user groups allowed to access the endpoint
+
+### Download Rules
+
+Download rules handle file access and retrieval. Configuration includes:
+
+- fileIdPosition: Specifies the position of the file ID in the URL path
+- Automatic validation of user access rights to requested files
+
+### Upload Rules
+
+Upload rules manage file upload functionality and restrictions. Configuration includes:
+
+- path: The target directory path for uploaded files
+- allowedTypes: Comma-separated list of allowed file extensions (e.g., jpg,png,pdf)
+- maxSize: Maximum allowed file size in megabytes
+
+### Locking Rules
+
+Locking rules provide exclusive access control for resources. Configuration includes:
+
+- action: The locking operation to perform
+  - lock: Lock a resource for exclusive access
+  - unlock: Release a previously locked resource
+- timeout: Duration in minutes before the lock automatically expires
+
+## Rule Validation
+
+When adding rules to an endpoint:
+
+- The rules array is initialized if it doesn't exist
+- Existing rule IDs are converted to strings
+- New rule IDs are validated before being added
+- The endpoint is revalidated after rule changes
+
+## Error Handling
+
+The rule addition process includes:
+
+- Validation that a rule is selected before saving
+- Error catching and display if the save fails
+- Loading state management during the save process
+- Success/error message display to the user

lib/Controller/EndpointsController.php

@@ -197,6 +197,8 @@ public function destroy(int $id): JSONResponse
 	 */
 	public function handlePath(string $_path): JSONResponse
 	{
+		// @todo: move to a rule service
+		/*
 		try {
 			$token = $this->request->getHeader('Authorization');
 			$this->authorizationService->authorize(authorization: $token);
@@ -206,6 +208,7 @@ public function handlePath(string $_path): JSONResponse
 				statusCode: 401
 			);
 		}
+		*/
 
 		// Find matching endpoints for the given path and method
 		$matchingEndpoints = $this->endpointMapper->findByPathRegex(