From de2382e25b78a4fee6726bd90214b2feb6b163a5 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Tue, 23 Jan 2024 11:40:54 +0100 Subject: [PATCH 1/3] No longer use url for sourceId in notifications / syncs when not needed --- api/src/Service/SynchronizationService.php | 93 ++++++++++++---------- 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/api/src/Service/SynchronizationService.php b/api/src/Service/SynchronizationService.php index b70a62ee0..4cfbd8048 100644 --- a/api/src/Service/SynchronizationService.php +++ b/api/src/Service/SynchronizationService.php @@ -14,6 +14,7 @@ use App\Exception\GatewayException; use CommonGateway\CoreBundle\Service\CallService; use CommonGateway\CoreBundle\Service\FileSystemHandleService; +use CommonGateway\CoreBundle\Service\GatewayResourceService; use CommonGateway\CoreBundle\Service\MappingService; use DateInterval; use DateTime; @@ -23,6 +24,7 @@ use Monolog\Logger; use Psr\Cache\CacheException; use Psr\Cache\InvalidArgumentException; +use Ramsey\Uuid\Uuid; use Safe\Exceptions\UrlException; use Symfony\Component\Console\Helper\TableSeparator; use Symfony\Component\Console\Style\SymfonyStyle; @@ -60,6 +62,7 @@ class SynchronizationService private SymfonyStyle $io; private Environment $twig; private MappingService $mappingService; + private GatewayResourceService $resourceService; private ActionEvent $event; private EventDispatcherInterface $eventDispatcher; @@ -83,6 +86,7 @@ class SynchronizationService * @param EventDispatcherInterface $eventDispatcher * @param MappingService $mappingService * @param FileSystemHandleService $fileSystemService + * @param GatewayResourceService $resourceService */ public function __construct( CallService $callService, @@ -98,7 +102,8 @@ public function __construct( Environment $twig, EventDispatcherInterface $eventDispatcher, MappingService $mappingService, - FileSystemHandleService $fileSystemService + FileSystemHandleService $fileSystemService, + GatewayResourceService $resourceService ) { $this->callService = $callService; $this->entityManager = $entityManager; @@ -119,6 +124,7 @@ public function __construct( $this->logger = new Logger('installation'); $this->mappingService = $mappingService; $this->fileSystemService = $fileSystemService; + $this->resourceService = $resourceService; } /** @@ -670,13 +676,15 @@ public function getSingleFromSource(Synchronization $synchronization): ?array $url = \Safe\parse_url($synchronization->getSource()->getLocation()); + $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $synchronization->getEndpoint().'/'.$synchronization->getSourceId(); + if ($url['scheme'] === 'http' || $url['scheme'] === 'https') { // Get object form source with callservice try { $this->logger->info("getSingleFromSource with Synchronization->sourceId = {$synchronization->getSourceId()}"); $response = $this->callService->call( $callServiceConfig['source'], - $synchronization->getEndpoint() ?? $callServiceConfig['endpoint'], + isset($this->configuration['location']) === true ? $callServiceConfig['endpoint'] : $endpoint, $callServiceConfig['method'] ?? 'GET', [ 'body' => '', @@ -694,7 +702,7 @@ public function getSingleFromSource(Synchronization $synchronization): ?array $result = $this->callService->decodeResponse($callServiceConfig['source'], $response); } elseif ($url['scheme'] === 'ftp') { // This only works if a file data equals a single Object(Entity). Or if the mapping on the Source or Synchronization results in data for just a single Object. - $result = $this->fileSystemService->call($synchronization->getSource(), $synchronization->getEndpoint() ?? $callServiceConfig['endpoint']); + $result = $this->fileSystemService->call($synchronization->getSource(), isset($this->configuration['location']) === true ? $callServiceConfig['endpoint'] : $endpoint); } $dot = new Dot($result); // The place where we can find the id field when looping through the list of objects, from $result root, by object (dot notation) @@ -711,15 +719,20 @@ public function getSingleFromSource(Synchronization $synchronization): ?array /** * Finds a synchronization object if it exists for the current object in the source, or creates one if it doesn't exist. * - * @param Source $source The source that is requested - * @param Entity $entity The entity that is requested - * @param string $sourceId The id of the object in the source + * @param Source $source The source that is requested + * @param Entity $entity The entity that is requested + * @param string $sourceId The id of the object in the source + * @param string|null $endpoint The endpoint of the synchronization. * * @return Synchronization|null A synchronization object related to the object in the source */ - public function findSyncBySource(Source $source, Entity $entity, string $sourceId): ?Synchronization + public function findSyncBySource(Source $source, Entity $entity, string $sourceId, ?string $endpoint = null): ?Synchronization { - $synchronization = $this->entityManager->getRepository('App:Synchronization')->findOneBy(['gateway' => $source, 'entity' => $entity, 'sourceId' => $sourceId]); + $criteria = ['gateway' => $source, 'entity' => $entity, 'sourceId' => $sourceId]; + if (empty($endpoint) === false) { + $criteria['endpoint'] = $endpoint; + } + $synchronization = $this->entityManager->getRepository('App:Synchronization')->findOneBy($criteria); if ($synchronization instanceof Synchronization) { if (isset($this->io)) { @@ -731,6 +744,7 @@ public function findSyncBySource(Source $source, Entity $entity, string $sourceI } $synchronization = new Synchronization($source, $entity); + $synchronization->setEndpoint($endpoint); $synchronization->setSourceId($sourceId); $this->entityManager->persist($synchronization); // We flush later @@ -1326,15 +1340,18 @@ private function syncToSource(Synchronization $synchronization, bool $existsInSo // $objectArray = $this->objectEntityService->checkGetObjectExceptions($data, $object, [], ['all' => true], 'application/ld+json'); // todo: maybe move this to foreach in getAllFromSource() (nice to have) - $callServiceConfig = $this->getCallServiceConfig($synchronization->getSource(), null, $objectArray); + $callServiceConfig = $this->getCallServiceConfig($synchronization->getSource(), $existsInSource ? $synchronization->getSourceId() : null, $objectArray); $objectArray = $this->mapOutput($objectArray); + $normalEndpoint = $synchronization->getEndpoint().($existsInSource ? '/'.$synchronization->getSourceId() : ''); + $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $normalEndpoint; + $objectString = $this->getObjectString($objectArray); try { $result = $this->callService->call( $callServiceConfig['source'], - $synchronization->getEndpoint() ?? $callServiceConfig['endpoint'], + isset($this->configuration['location']) === true ? $callServiceConfig['endpoint'] : $endpoint, $callServiceConfig['method'] ?? ($existsInSource ? 'PUT' : 'POST'), [ 'body' => $objectString, @@ -1469,41 +1486,35 @@ private function syncThroughComparing(Synchronization $synchronization): Synchro */ public function aquireObject(string $url, Entity $entity): ?ObjectEntity { - // 1. Get the domain from the url - $parse = \Safe\parse_url($url); - $location = $parse['scheme'].'://'.$parse['host']; - - // 2.c Try to establish a source for the domain - $source = $this->entityManager->getRepository('App:Gateway')->findOneBy(['location'=>$location]); - - // 2.b The source might be on a path e.g. /v1 so if whe cant find a source let try to cycle - if ($source instanceof Source === false && isset($parse['path']) === true) { - foreach (explode('/', $parse['path']) as $pathPart) { - if ($pathPart !== '') { - $location = $location.'/'.$pathPart; - } - $source = $this->entityManager->getRepository('App:Gateway')->findOneBy(['location'=>$location]); - if ($source !== null) { - break; - } - } - } - if ($source instanceof Source === false) { - return null; - } - - // 3 If we have a source we can establish an endpoint. - $endpoint = str_replace($location, '', $url); - - // 4 Create sync - $synchronization = new Synchronization($source, $entity); - $synchronization->setSourceId($url); - $synchronization->setEndpoint($endpoint); + $source = $this->resourceService->findSourceForUrl($url, 'conduction-nl/commonground-gateway', $endpoint); + $sourceId = $this->getSourceId($endpoint, $url); - $this->entityManager->persist($synchronization); + $synchronization = $this->findSyncBySource($source, $entity, $sourceId, $endpoint); $this->synchronize($synchronization); return $synchronization->getObject(); } + + /** + * A function best used after resourceService->findSourceForUrl and/or before $this->findSyncBySource. + * This function will get the uuid / int id from the end of an endpoint. This is the sourceId for a Synchronization. + * + * @param string $endpoint The endpoint to get the SourceId from. + * @param string|null $url The url used as back-up for SourceId if no proper SourceId can be found. + * + * @return string|null The sourceId, will be equal to $url if end part of the endpoint isn't an uuid or integer. + */ + public function getSourceId(string &$endpoint, ?string $url = null): ?string + { + $explodedEndpoint = explode('/', $endpoint); + $sourceId = end($explodedEndpoint); + if (Uuid::isValid($sourceId) === true || is_int((int) $sourceId) === true) { + $endpoint = str_replace("/$sourceId", '', $endpoint); + } else { + $sourceId = $url; + } + + return $sourceId; + } } From 1ffc507873927d15cf0bb065b1bb532ba2d325e4 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Tue, 23 Jan 2024 11:52:48 +0100 Subject: [PATCH 2/3] Small fix & code cleanup for setting Synchronization Endpoint --- api/src/Service/SynchronizationService.php | 23 ++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/api/src/Service/SynchronizationService.php b/api/src/Service/SynchronizationService.php index 4cfbd8048..cc1fb43e0 100644 --- a/api/src/Service/SynchronizationService.php +++ b/api/src/Service/SynchronizationService.php @@ -309,7 +309,7 @@ private function loopThroughCollectionResults(array $results, array $config): ar array_key_exists('object', $this->configuration['apiSource']['location']) && $result = $dot->get($this->configuration['apiSource']['location']['object'], $result); // Lets grab the sync object, if we don't find an existing one, this will create a new one: - $synchronization = $this->findSyncBySource($config['source'], $config['entity'], $id); + $synchronization = $this->findSyncBySource($config['source'], $config['entity'], $id, $this->configuration['location'] ?? null); // todo: Another search function for sync object. If no sync object is found, look for matching properties... // todo: ...in $result and an ObjectEntity in db. And then create sync for an ObjectEntity if we find one this way. (nice to have) // Other option to find a sync object, currently not used: @@ -676,7 +676,13 @@ public function getSingleFromSource(Synchronization $synchronization): ?array $url = \Safe\parse_url($synchronization->getSource()->getLocation()); - $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $synchronization->getEndpoint().'/'.$synchronization->getSourceId(); + if (isset($this->configuration['location']) === true) { + $endpoint = $this->configuration['location']; + $synchronization->setEndpoint($endpoint); + } else { + $normalEndpoint = $synchronization->getEndpoint().'/'.$synchronization->getSourceId(); + $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $normalEndpoint; + } if ($url['scheme'] === 'http' || $url['scheme'] === 'https') { // Get object form source with callservice @@ -684,7 +690,7 @@ public function getSingleFromSource(Synchronization $synchronization): ?array $this->logger->info("getSingleFromSource with Synchronization->sourceId = {$synchronization->getSourceId()}"); $response = $this->callService->call( $callServiceConfig['source'], - isset($this->configuration['location']) === true ? $callServiceConfig['endpoint'] : $endpoint, + $endpoint, $callServiceConfig['method'] ?? 'GET', [ 'body' => '', @@ -1343,15 +1349,20 @@ private function syncToSource(Synchronization $synchronization, bool $existsInSo $callServiceConfig = $this->getCallServiceConfig($synchronization->getSource(), $existsInSource ? $synchronization->getSourceId() : null, $objectArray); $objectArray = $this->mapOutput($objectArray); - $normalEndpoint = $synchronization->getEndpoint().($existsInSource ? '/'.$synchronization->getSourceId() : ''); - $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $normalEndpoint; + if (isset($this->configuration['location']) === true) { + $endpoint = $this->configuration['location']; + $synchronization->setEndpoint($endpoint); + } else { + $normalEndpoint = $synchronization->getEndpoint().($existsInSource ? '/'.$synchronization->getSourceId() : ''); + $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $normalEndpoint; + } $objectString = $this->getObjectString($objectArray); try { $result = $this->callService->call( $callServiceConfig['source'], - isset($this->configuration['location']) === true ? $callServiceConfig['endpoint'] : $endpoint, + $endpoint, $callServiceConfig['method'] ?? ($existsInSource ? 'PUT' : 'POST'), [ 'body' => $objectString, From a1e183c455823412c8852c25314cc71198dd2877 Mon Sep 17 00:00:00 2001 From: Wilco Louwerse Date: Tue, 23 Jan 2024 12:30:51 +0100 Subject: [PATCH 3/3] Remove else statements and 1 line if statements --- api/src/Service/SynchronizationService.php | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/api/src/Service/SynchronizationService.php b/api/src/Service/SynchronizationService.php index cc1fb43e0..481f56ede 100644 --- a/api/src/Service/SynchronizationService.php +++ b/api/src/Service/SynchronizationService.php @@ -676,12 +676,13 @@ public function getSingleFromSource(Synchronization $synchronization): ?array $url = \Safe\parse_url($synchronization->getSource()->getLocation()); + $endpoint = $synchronization->getEndpoint().'/'.$synchronization->getSourceId(); + if (str_contains('http', $synchronization->getSourceId()) === true) { + $endpoint = $synchronization->getEndpoint(); + } if (isset($this->configuration['location']) === true) { $endpoint = $this->configuration['location']; $synchronization->setEndpoint($endpoint); - } else { - $normalEndpoint = $synchronization->getEndpoint().'/'.$synchronization->getSourceId(); - $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $normalEndpoint; } if ($url['scheme'] === 'http' || $url['scheme'] === 'https') { @@ -1349,12 +1350,16 @@ private function syncToSource(Synchronization $synchronization, bool $existsInSo $callServiceConfig = $this->getCallServiceConfig($synchronization->getSource(), $existsInSource ? $synchronization->getSourceId() : null, $objectArray); $objectArray = $this->mapOutput($objectArray); + $endpoint = $synchronization->getEndpoint(); + if ($existsInSource === true) { + $endpoint = $endpoint.'/'.$synchronization->getSourceId(); + } + if (str_contains('http', $synchronization->getSourceId()) === true) { + $endpoint = $synchronization->getEndpoint(); + } if (isset($this->configuration['location']) === true) { $endpoint = $this->configuration['location']; $synchronization->setEndpoint($endpoint); - } else { - $normalEndpoint = $synchronization->getEndpoint().($existsInSource ? '/'.$synchronization->getSourceId() : ''); - $endpoint = str_contains('http', $synchronization->getSourceId()) === true ? $synchronization->getEndpoint() : $normalEndpoint; } $objectString = $this->getObjectString($objectArray);