diff --git a/modules/datastore/datastore.services.yml b/modules/datastore/datastore.services.yml index 3aef3143c4..54b110352d 100644 --- a/modules/datastore/datastore.services.yml +++ b/modules/datastore/datastore.services.yml @@ -14,8 +14,14 @@ services: dkan.datastore.service.post_import: class: \Drupal\datastore\Service\PostImport arguments: - - '@database' + - '@config.factory' + - '@dkan.datastore.logger_channel' - '@dkan.metastore.resource_mapper' + - '@dkan.datastore.service.resource_processor_collector' + - '@dkan.metastore.data_dictionary_discovery' + - '@dkan.metastore.reference_lookup' + - '@dkan.datastore.service' + - '@database' dkan.datastore.query: class: \Drupal\datastore\Service\Query diff --git a/modules/datastore/src/Plugin/QueueWorker/PostImportResourceProcessor.php b/modules/datastore/src/Plugin/QueueWorker/PostImportResourceProcessor.php index 05a4bb0d59..a2e33ff575 100644 --- a/modules/datastore/src/Plugin/QueueWorker/PostImportResourceProcessor.php +++ b/modules/datastore/src/Plugin/QueueWorker/PostImportResourceProcessor.php @@ -5,17 +5,7 @@ use Drupal\Core\Config\ConfigFactoryInterface; use Drupal\Core\Plugin\ContainerFactoryPluginInterface; use Drupal\Core\Queue\QueueWorkerBase; -use Drupal\common\DataResource; -use Drupal\datastore\DataDictionary\AlterTableQueryBuilderInterface; -use Drupal\datastore\PostImportResult; -use Drupal\datastore\DatastoreService; use Drupal\datastore\Service\PostImport; -use Drupal\datastore\Service\ResourceProcessor\ResourceDoesNotHaveDictionary; -use Drupal\datastore\Service\ResourceProcessorCollector; -use Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface; -use Drupal\metastore\Reference\ReferenceLookup; -use Drupal\metastore\ResourceMapper; -use Psr\Log\LoggerInterface; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -32,54 +22,22 @@ */ class PostImportResourceProcessor extends QueueWorkerBase implements ContainerFactoryPluginInterface { - /** - * The datastore.settings config. - * - * @var \Drupal\Core\Config\ImmutableConfig - */ - protected $config; - - /** - * A logger channel for this plugin. - */ - protected LoggerInterface $logger; - - /** - * The metastore resource mapper service. - */ - protected ResourceMapper $resourceMapper; - - /** - * The resource processor collector service. - */ - protected ResourceProcessorCollector $resourceProcessorCollector; - - /** - * The datastore service. - */ - protected DatastoreService $datastoreService; - /** * The PostImport service. - */ - protected PostImport $postImport; - - /** - * Data dictionary discovery service. * - * @var \Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface + * @var \Drupal\datastore\Service\PostImport */ - protected $dataDictionaryDiscovery; + protected PostImport $postImport; /** - * Reference lookup service. + * The datastore.settings config. * - * @var \Drupal\metastore\Reference\ReferenceLookup + * @var \Drupal\Core\Config\ImmutableConfig */ - protected $referenceLookup; + protected $configFactory; /** - * Build queue worker. + * Constructor for PostImportResourceProcessor. * * @param array $configuration * A configuration array containing information about the plugin instance. @@ -87,53 +45,21 @@ class PostImportResourceProcessor extends QueueWorkerBase implements ContainerFa * The plugin_id for the plugin instance. * @param mixed $plugin_definition * The plugin implementation definition. - * @param \Drupal\Core\Config\ConfigFactoryInterface $configFactory - * The config.factory service. - * @param \Drupal\datastore\DataDictionary\AlterTableQueryBuilderInterface $alter_table_query_builder - * The alter table query factory service. - * @param \Psr\Log\LoggerInterface $logger_channel - * A logger channel factory instance. - * @param \Drupal\metastore\ResourceMapper $resource_mapper - * The metastore resource mapper service. - * @param \Drupal\datastore\Service\ResourceProcessorCollector $processor_collector - * The resource processor collector service. - * @param \Drupal\datastore\DatastoreService $datastoreService - * The resource datastore service. * @param \Drupal\datastore\Service\PostImport $post_import - * The post import service. - * @param \Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface $data_dictionary_discovery - * The data-dictionary discovery service. - * @param \Drupal\metastore\Reference\ReferenceLookup $referenceLookup - * The reference lookup service. + * The PostImport service. + * @param \Drupal\Core\Config\ConfigFactoryInterface $config_factory + * The config.factory service. */ public function __construct( array $configuration, $plugin_id, $plugin_definition, - ConfigFactoryInterface $configFactory, - AlterTableQueryBuilderInterface $alter_table_query_builder, - LoggerInterface $logger_channel, - ResourceMapper $resource_mapper, - ResourceProcessorCollector $processor_collector, - DatastoreService $datastoreService, PostImport $post_import, - DataDictionaryDiscoveryInterface $data_dictionary_discovery, - ReferenceLookup $referenceLookup + ConfigFactoryInterface $config_factory, ) { parent::__construct($configuration, $plugin_id, $plugin_definition); - $this->config = $configFactory; - $this->logger = $logger_channel; - $this->resourceMapper = $resource_mapper; - $this->resourceProcessorCollector = $processor_collector; - $this->datastoreService = $datastoreService; $this->postImport = $post_import; - $this->dataDictionaryDiscovery = $data_dictionary_discovery; - // Set the timeout for database connections to the queue lease time. - // This ensures that database connections will remain open for the - // duration of the time the queue is being processed. - $timeout = (int) $plugin_definition['cron']['lease_time']; - $alter_table_query_builder->setConnectionTimeout($timeout); - $this->referenceLookup = $referenceLookup; + $this->configFactory = $config_factory; } /** @@ -144,15 +70,8 @@ public static function create(ContainerInterface $container, array $configuratio $configuration, $plugin_id, $plugin_definition, - $container->get('config.factory'), - $container->get('dkan.datastore.data_dictionary.alter_table_query_builder.mysql'), - $container->get('dkan.datastore.logger_channel'), - $container->get('dkan.metastore.resource_mapper'), - $container->get('dkan.datastore.service.resource_processor_collector'), - $container->get('dkan.datastore.service'), $container->get('dkan.datastore.service.post_import'), - $container->get('dkan.metastore.data_dictionary_discovery'), - $container->get('dkan.metastore.reference_lookup'), + $container->get('config.factory'), ); } @@ -160,115 +79,8 @@ public static function create(ContainerInterface $container, array $configuratio * {@inheritdoc} */ public function processItem($data) { - $postImportResult = $this->postImportProcessItem($data); - $drop_config = $this->config->get('datastore.settings') - ->get('drop_datastore_on_post_import_error'); - - if ($postImportResult->getPostImportStatus() === 'done') { - $this->invalidateCacheTags(DataResource::buildUniqueIdentifier( - $data->getIdentifier(), - $data->getVersion(), - DataResource::DEFAULT_SOURCE_PERSPECTIVE - )); - } - if ($postImportResult->getPostImportStatus() === 'error' && $drop_config) { - $identifier = $data->getIdentifier(); - try { - $this->datastoreService->drop($identifier, NULL, FALSE); - $this->logger->notice('Successfully dropped the datastore for resource @identifier due to a post import error. Visit the Datastore Import Status dashboard for details.', [ - '@identifier' => $identifier, - ]); - } - catch (\Exception $e) { - $this->logger->error($e->getMessage()); - } - } - // Store the results of the PostImportResult object. - $postImportResult->storeResult(); - } - - /** - * Pass along new resource to resource processors. - * - * @todo This method should not contain references to data dictionary - * behavior. Put all the dictionary-related logic into - * DictionaryEnforcer::process(). - * - * @param \Drupal\common\DataResource $resource - * DKAN Resource. - */ - public function postImportProcessItem(DataResource $resource): PostImportResult { - $latest_resource = $this->resourceMapper->get($resource->getIdentifier()); - - // Stop if resource no longer exists. - if (!isset($latest_resource)) { - $this->logger->notice('Cancelling resource processing; resource no longer exists.'); - return $this->createPostImportResult('error', 'Cancelling resource processing; resource no longer exists.', $resource); - } - // Stop if resource has changed. - if ($resource->getVersion() !== $latest_resource->getVersion()) { - $this->logger->notice('Cancelling resource processing; resource has changed.'); - return $this->createPostImportResult('error', 'Cancelling resource processing; resource has changed.', $resource); - } - - try { - // Run all tagged resource processors. - $processors = $this->resourceProcessorCollector->getResourceProcessors(); - - if (DataDictionaryDiscoveryInterface::MODE_NONE === $this->dataDictionaryDiscovery->getDataDictionaryMode()) { - $postImportResult = $this->createPostImportResult('N/A', 'Data-Dictionary Disabled', $resource); - } - else { - array_map(fn ($processor) => $processor->process($resource), $processors); - $postImportResult = $this->createPostImportResult('done', NULL, $resource); - $this->logger->notice('Post import job for resource @id completed.', ['@id' => (string) $resource->getIdentifier()]); - } - } - catch (ResourceDoesNotHaveDictionary $e) { - // ResourceDoesNotHaveDictionary means there was no data dictionary for - // the given resource. This is not an error because not all resources have - // data dictionaries, but we should tell the user in case they think the - // resource should have one. - // @see \Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer::getDataDictionaryForResource() - $this->logger->notice($e->getMessage()); - $postImportResult = $this->createPostImportResult('done', 'Resource ' . $e->getResourceId() . ' does not have a data dictionary.', $resource); - } - catch (\Exception $e) { - // General catch-all for errors. - $this->logger->error($e->getMessage()); - $postImportResult = $this->createPostImportResult('error', $e->getMessage(), $resource); - } - - return $postImportResult; - } - - /** - * Invalidate all appropriate cache tags for this resource. - * - * @param mixed $resourceId - * A resource ID. - */ - protected function invalidateCacheTags(mixed $resourceId) { - $this->referenceLookup->invalidateReferencerCacheTags('distribution', $resourceId, 'downloadURL'); - } - - /** - * Create the PostImportResult object. - * - * @param string $status - * Status of the post import process. - * @param string $message - * Error messages retrieved during the post import process. - * @param \Drupal\common\DataResource $resource - * The DKAN resource being imported. - */ - private function createPostImportResult($status, $message, DataResource $resource): PostImportResult { - return new PostImportResult([ - 'resource_identifier' => $resource->getIdentifier(), - 'resourceVersion' => $resource->getVersion(), - 'postImportStatus' => $status, - 'postImportMessage' => $message, - ], $this->resourceMapper, $this->postImport); + $result = $this->postImport->processResource($data); + $result->storeResult(); } } diff --git a/modules/datastore/src/PostImportResult.php b/modules/datastore/src/PostImportResult.php index a20cd6c5f9..27def4b6cb 100644 --- a/modules/datastore/src/PostImportResult.php +++ b/modules/datastore/src/PostImportResult.php @@ -9,7 +9,7 @@ /** * PostImportResult class to create PostImportResult objects. * - * Contains the results of the PostImportResourceProcessor. + * Contains the results of the PostImport. */ class PostImportResult { @@ -66,14 +66,12 @@ class PostImportResult { */ public function __construct( $postImportResult, - ResourceMapper $resourceMapper, PostImport $postImport ) { $this->resourceIdentifier = $postImportResult['resource_identifier']; $this->resourceVersion = $postImportResult['resourceVersion']; $this->postImportStatus = $postImportResult['postImportStatus']; $this->postImportMessage = $postImportResult['postImportMessage']; - $this->resourceMapper = $resourceMapper; $this->postImport = $postImport; } diff --git a/modules/datastore/src/Service/PostImport.php b/modules/datastore/src/Service/PostImport.php index 83402287b1..3247974ad7 100644 --- a/modules/datastore/src/Service/PostImport.php +++ b/modules/datastore/src/Service/PostImport.php @@ -2,40 +2,252 @@ namespace Drupal\datastore\Service; -use Drupal\Core\Database\Connection; +use Drupal\common\DataResource; +use Drupal\datastore\DatastoreService; +use Drupal\Core\Config\ConfigFactoryInterface; +use Drupal\datastore\PostImportResult; +use Drupal\datastore\Service\ResourceProcessor\ResourceDoesNotHaveDictionary; +use Drupal\datastore\Service\ResourceProcessorCollector; +use Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface; +use Drupal\metastore\Reference\ReferenceLookup; use Drupal\metastore\ResourceMapper; +use Psr\Log\LoggerInterface; +use Drupal\Core\Database\Connection; /** - * PostImport status storage service. + * Service to handle post-import resource processing. */ class PostImport { /** - * The database connection. + * The config factory. * - * @var \Drupal\Core\Database\Connection + * @var \Drupal\Core\Config\ConfigFactoryInterface */ - protected $connection; + protected ConfigFactoryInterface $configFactory; /** - * The metastore resource mapper service. + * The logger service. + * + * @var \Psr\Log\LoggerInterface + */ + protected LoggerInterface $logger; + + /** + * The resource mapper. + * + * @var \Drupal\metastore\ResourceMapper */ protected ResourceMapper $resourceMapper; /** - * Constructs a new PostImport object. + * The resource processor collector. + * + * @var \Drupal\datastore\Service\ResourceProcessorCollector + */ + protected ResourceProcessorCollector $resourceProcessorCollector; + + /** + * The data dictionary discovery interface. * + * @var \Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface + */ + protected DataDictionaryDiscoveryInterface $dataDictionaryDiscovery; + + /** + * The reference lookup service. + * + * @var \Drupal\metastore\Reference\ReferenceLookup + */ + protected ReferenceLookup $referenceLookup; + + /** + * The post import result service. + * + * @var \Drupal\datastore\PostImportResult + */ + protected PostImportResult $postImportResult; + + /** + * The datastore service. + * + * @var \Drupal\datastore\DatastoreService + */ + protected DatastoreService $datastoreService; + + /** + * The database connection. + * + * @var \Drupal\Core\Database\Connection + */ + protected Connection $connection; + + /** + * Constructs a new PostImport service. + * + * @param \Drupal\Core\Config\ConfigFactoryInterface $configFactory + * The config factory service. + * @param \Psr\Log\LoggerInterface $logger + * The logger service. + * @param \Drupal\metastore\ResourceMapper $resourceMapper + * The resource mapper service. + * @param \Drupal\datastore\Service\ResourceProcessorCollector $resourceProcessorCollector + * The resource processor collector service. + * @param \Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface $dataDictionaryDiscovery + * The data dictionary discovery interface. + * @param \Drupal\metastore\Reference\ReferenceLookup $referenceLookup + * The reference lookup service. + * @param \Drupal\datastore\DatastoreService $datastoreService + * The datastore service. * @param \Drupal\Core\Database\Connection $connection * The database connection. - * @param \Drupal\metastore\ResourceMapper $resource_mapper - * The metastore resource mapper service. */ public function __construct( + ConfigFactoryInterface $configFactory, + LoggerInterface $logger, + ResourceMapper $resourceMapper, + ResourceProcessorCollector $resourceProcessorCollector, + DataDictionaryDiscoveryInterface $dataDictionaryDiscovery, + ReferenceLookup $referenceLookup, + DatastoreService $datastoreService, Connection $connection, - ResourceMapper $resource_mapper - ) { + ) { + $this->configFactory = $configFactory; + $this->logger = $logger; + $this->resourceMapper = $resourceMapper; + $this->resourceProcessorCollector = $resourceProcessorCollector; + $this->dataDictionaryDiscovery = $dataDictionaryDiscovery; + $this->referenceLookup = $referenceLookup; + $this->datastoreService = $datastoreService; $this->connection = $connection; - $this->resourceMapper = $resource_mapper; + } + + /** + * Pass along new resource to resource processors. + * + * @todo This method should not contain references to data dictionary + * behavior. Put all the dictionary-related logic into + * DictionaryEnforcer::process(). + * + * @param \Drupal\common\DataResource $resource + * DKAN Resource. + * + * @return \Drupal\datastore\PostImportResult + * The post import result service. + */ + public function processResource(DataResource $resource): PostImportResult { + if ($result = $this->validateResource($resource)) { + return $result; + } + + try { + $this->processResourceProcessors($resource); + $this->logger->notice('Post import job for resource @id completed.', ['@id' => $resource->getIdentifier()]); + $this->invalidateCacheTags($resource->getIdentifier()); + return $this->createPostImportResult('done', NULL, $resource); + } + catch (ResourceDoesNotHaveDictionary $e) { + + $this->logger->notice($e->getMessage()); + return $this->createPostImportResult('done', 'Resource does not have a data dictionary.', $resource); + } + catch (\Exception $e) { + + $this->handleProcessingError($resource, $e); + return $this->createPostImportResult('error', $e->getMessage(), $resource); + } + } + + /** + * Handle errors during resource processing. + * + * @param \Drupal\common\DataResource $resource + * DKAN Resource. + * @param \Exception $exception + * The caught exception. + */ + private function handleProcessingError(DataResource $resource, \Exception $exception): void { + $identifier = $resource->getIdentifier(); + + if ($this->configFactory->get('datastore.settings')->get('drop_datastore_on_post_import_error')) { + try { + $this->drop($identifier, NULL, FALSE); + $this->logger->notice('Successfully dropped the datastore for resource @identifier due to a post import error. Visit the Datastore Import Status dashboard for details.', [ + '@identifier' => $identifier, + ]); + } + catch (\Exception $dropException) { + $this->logger->error($dropException->getMessage()); + } + } + + $this->logger->error($exception->getMessage()); + } + + /** + * Process resource. + * + * @param \Drupal\common\DataResource $resource + * DKAN Resource. + * + * @throws \Exception + */ + private function processResourceProcessors(DataResource $resource): void { + $processors = $this->resourceProcessorCollector->getResourceProcessors(); + array_map(fn($processor) => $processor->process($resource), $processors); + } + + /** + * Validation checks before processing resource. + * + * @param \Drupal\common\DataResource $resource + * DKAN Resource. + * + * @return \Drupal\datastore\PostImportResult|null + * Post import result if validation fails, or NULL if validation passes. + */ + private function validateResource(DataResource $resource): ?PostImportResult { + $latestResource = $this->resourceMapper->get($resource->getIdentifier()); + + if (!$latestResource) { + $this->logger->notice('Cancelling resource processing; resource no longer exists.'); + return $this->createPostImportResult('error', 'Cancelling resource processing; resource no longer exists.', $resource); + } + + if ($resource->getVersion() !== $latestResource->getVersion()) { + $this->logger->notice('Cancelling resource processing; resource has changed.'); + return $this->createPostImportResult('error', 'Cancelling resource processing; resource has changed.', $resource); + } + + if ($this->dataDictionaryDiscovery->getDataDictionaryMode() === DataDictionaryDiscoveryInterface::MODE_NONE) { + $this->logger->notice('Data-Dictionary Disabled'); + return $this->createPostImportResult('N/A', 'Data-Dictionary Disabled', $resource); + } + + return NULL; + } + + /** + * Create the PostImportResult object. + * + * @param string $status + * Status of the post import process. + * @param string $message + * Error messages retrieved during the post import process. + * @param \Drupal\common\DataResource $resource + * The DKAN resource being imported. + * + * @return \Drupal\datastore\PostImportResult + * The post import result service. + */ + protected function createPostImportResult($status, $message, DataResource $resource): PostImportResult { + return new PostImportResult([ + 'resource_identifier' => $resource->getIdentifier(), + 'resourceVersion' => $resource->getVersion(), + 'postImportStatus' => $status, + 'postImportMessage' => $message, + ], + $this); } /** @@ -116,4 +328,30 @@ public function removeJobStatus($resourceIdentifier): bool { } } + /** + * Remove row. + * + * @param string $resourceIdentifier + * The resource identifier of the distribution. + */ + public function drop($resourceIdentifier): bool { + try { + $this->datastoreService->drop($resourceIdentifier, NULL, FALSE); + return TRUE; + } + catch (\Exception $e) { + throw $e; + } + } + + /** + * Invalidate all appropriate cache tags for this resource. + * + * @param mixed $resourceId + * A resource ID. + */ + public function invalidateCacheTags($resourceId): void { + $this->referenceLookup->invalidateReferencerCacheTags('distribution', $resourceId, 'downloadURL'); + } + } diff --git a/modules/datastore/tests/src/Kernel/Plugin/QueueWorker/PostImportResourceProcessorTest.php b/modules/datastore/tests/src/Kernel/Plugin/QueueWorker/PostImportResourceProcessorTest.php deleted file mode 100644 index df8f2d15d0..0000000000 --- a/modules/datastore/tests/src/Kernel/Plugin/QueueWorker/PostImportResourceProcessorTest.php +++ /dev/null @@ -1,256 +0,0 @@ -config('metastore.settings') - ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_REFERENCE) - ->save(); - - // Mock the resource mapper to return a given data resource with no - // describedBy property. - $resource = new DataResource('test.csv', 'text/csv'); - $resource_mapper = $this->getMockBuilder(ResourceMapper::class) - ->disableOriginalConstructor() - ->onlyMethods(['get']) - ->getMock(); - $resource_mapper->expects($this->once()) - ->method('get') - ->willReturn($resource); - $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); - - // Mock the dictionary enforcer to throw an exception so that we can avoid - // node type dependenies. - $no_dictionary_exception = new ResourceDoesNotHaveDictionary('test', 123); - $enforcer = $this->getMockBuilder(DictionaryEnforcer::class) - ->disableOriginalConstructor() - ->onlyMethods(['process']) - ->getMock(); - $enforcer->expects($this->once()) - ->method('process') - ->willThrowException($no_dictionary_exception); - $this->container->set('dkan.datastore.service.resource_processor.dictionary_enforcer', $enforcer); - - // Create a post import processor. - /** @var \Drupal\datastore\Plugin\QueueWorker\PostImportResourceProcessor $processor */ - $processor = PostImportResourceProcessor::create( - $this->container, - [], - 'post_import', - [ - 'cron' => [ - 'time' => 180, - 'lease_time' => 10800, - ], - ] - ); - - // The results of post import processing should reflect that the resource - // does not have a data dictionary. - $result = $processor->postImportProcessItem($resource); - $this->assertEquals( - 'Resource test does not have a data dictionary.', - $result->getPostImportMessage() - ); - $this->assertEquals( - 'done', - $result->getPostImportStatus() - ); - } - - /** - * @covers ::processItem - */ - public function testProcessItem() { - $data_identifier = 'test_identifier'; - - $this->config('datastore.settings') - ->set('drop_datastore_on_post_import_error', TRUE) - ->save(); - - // Our error result. - $error_result = $this->getMockBuilder(PostImportResult::class) - ->disableOriginalConstructor() - ->onlyMethods(['getPostImportStatus', 'storeResult']) - ->getMock(); - $error_result->expects($this->any()) - ->method('getPostImportStatus') - ->willReturn('error'); - $error_result->expects($this->once()) - ->method('storeResult'); - - // Mock a logger to expect error logging. - $logger = $this->getMockBuilder(LoggerChannelInterface::class) - ->onlyMethods(['error', 'notice']) - ->getMockForAbstractClass(); - // Never expect an error. - $logger->expects($this->never()) - ->method('error'); - // Expect one notice. - $logger->expects($this->once()) - ->method('notice') - ->with( - 'Successfully dropped the datastore for resource @identifier due to a post import error. Visit the Datastore Import Status dashboard for details.', - ['@identifier' => $data_identifier], - ); - $this->container->set('dkan.datastore.logger_channel', $logger); - - // Datastore service will always succeed. Mocked so we don't have to deal - // with dropping an actual datastore. - $datastore_service = $this->getMockBuilder(DatastoreService::class) - ->disableOriginalConstructor() - ->onlyMethods(['drop']) - ->getMock(); - $datastore_service->expects($this->once()) - ->method('drop'); - // Put the service into the service container. - $this->container->set('dkan.datastore.service', $datastore_service); - - // Return our error result. - $post_import_resource_processor = $this->getMockBuilder(PostImportResourceProcessor::class) - ->setConstructorArgs([ - [], - '', - ['cron' => ['lease_time' => 10800]], - $this->container->get('config.factory'), - $this->container->get('dkan.datastore.data_dictionary.alter_table_query_builder.mysql'), - $this->container->get('dkan.datastore.logger_channel'), - $this->container->get('dkan.metastore.resource_mapper'), - $this->container->get('dkan.datastore.service.resource_processor_collector'), - $this->container->get('dkan.datastore.service'), - $this->container->get('dkan.datastore.service.post_import'), - $this->container->get('dkan.metastore.data_dictionary_discovery'), - $this->container->get('dkan.metastore.reference_lookup'), - ]) - ->onlyMethods(['postImportProcessItem']) - ->getMock(); - $post_import_resource_processor->expects($this->once()) - ->method('postImportProcessItem') - ->willReturn($error_result); - - // Data we'll pass to our method under test. - $data = $this->getMockBuilder(DataResource::class) - ->disableOriginalConstructor() - ->onlyMethods(['getIdentifier']) - ->getMock(); - $data->expects($this->once()) - ->method('getIdentifier') - ->willReturn($data_identifier); - - $post_import_resource_processor->processItem($data); - } - - /** - * @covers ::processItem - */ - public function testProcessItemExceptionPath() { - $this->config('datastore.settings') - ->set('drop_datastore_on_post_import_error', TRUE) - ->save(); - - // Our error result. - $error_result = $this->getMockBuilder(PostImportResult::class) - ->disableOriginalConstructor() - ->onlyMethods(['getPostImportStatus', 'storeResult']) - ->getMock(); - $error_result->expects($this->any()) - ->method('getPostImportStatus') - ->willReturn('error'); - $error_result->expects($this->once()) - ->method('storeResult'); - - // Mock a logger to expect error logging. - $logger = $this->getMockBuilder(LoggerChannelInterface::class) - ->onlyMethods(['error', 'notice']) - ->getMockForAbstractClass(); - // Expect an error. - $logger->expects($this->once()) - ->method('error'); - // Expect no notices. - $logger->expects($this->never()) - ->method('notice'); - $this->container->set('dkan.datastore.logger_channel', $logger); - - // Datastore service rigged to explode. - $datastore_service = $this->getMockBuilder(DatastoreService::class) - ->disableOriginalConstructor() - ->onlyMethods(['drop']) - ->getMock(); - $datastore_service->expects($this->once()) - ->method('drop') - ->willThrowException(new \Exception('our test message')); - // Put the service into the service container. - $this->container->set('dkan.datastore.service', $datastore_service); - - // Return our error result. - $post_import_resource_processor = $this->getMockBuilder(PostImportResourceProcessor::class) - ->setConstructorArgs([ - [], - '', - ['cron' => ['lease_time' => 10800]], - $this->container->get('config.factory'), - $this->container->get('dkan.datastore.data_dictionary.alter_table_query_builder.mysql'), - $this->container->get('dkan.datastore.logger_channel'), - $this->container->get('dkan.metastore.resource_mapper'), - $this->container->get('dkan.datastore.service.resource_processor_collector'), - $this->container->get('dkan.datastore.service'), - $this->container->get('dkan.datastore.service.post_import'), - $this->container->get('dkan.metastore.data_dictionary_discovery'), - $this->container->get('dkan.metastore.reference_lookup'), - ]) - ->onlyMethods(['postImportProcessItem']) - ->getMock(); - $post_import_resource_processor->expects($this->once()) - ->method('postImportProcessItem') - ->willReturn($error_result); - - // Data we'll pass to our method under test. - $data = $this->getMockBuilder(DataResource::class) - ->disableOriginalConstructor() - ->onlyMethods(['getIdentifier']) - ->getMock(); - $data->expects($this->once()) - ->method('getIdentifier') - ->willReturn('test'); - - $post_import_resource_processor->processItem($data); - } - -} diff --git a/modules/datastore/tests/src/Kernel/Service/PostImportTest.php b/modules/datastore/tests/src/Kernel/Service/PostImportTest.php index a69b85fa6e..e61a439dad 100644 --- a/modules/datastore/tests/src/Kernel/Service/PostImportTest.php +++ b/modules/datastore/tests/src/Kernel/Service/PostImportTest.php @@ -2,9 +2,17 @@ namespace Drupal\Tests\datastore\Kernel\Service; -use Drupal\Core\Database\Connection; +use Drupal\common\DataResource; +use Drupal\Core\Logger\LoggerChannelInterface; +use Drupal\datastore\DatastoreService; +use Drupal\datastore\PostImportResult; +use Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer; +use Drupal\datastore\Service\ResourceProcessor\ResourceDoesNotHaveDictionary; use Drupal\KernelTests\KernelTestBase; +use Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface; +use Drupal\metastore\ResourceMapper; use Drupal\datastore\Service\PostImport; +use Drupal\Core\Database\Connection; /** * Tests the PostImport service. @@ -24,6 +32,432 @@ class PostImportTest extends KernelTestBase { 'metastore', ]; + /** + * @covers ::processResource + */ + public function testProcessResourceChangedResource() { + $this->installEntitySchema('resource_mapping'); + + $this->config('metastore.settings') + ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->save(); + + $resource_a = new DataResource('test.csv', 'text/csv'); + + $resource_b = (new DataResource('test2.csv', 'text/csv'))->createNewVersion(); + + $resource_mapper = $this->getMockBuilder(ResourceMapper::class) + ->disableOriginalConstructor() + ->onlyMethods(['get']) + ->getMock(); + + $resource_mapper->expects($this->once()) + ->method('get') + ->willReturn($resource_a); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + // Mock a logger to expect error logging. + $logger = $this->getMockBuilder(LoggerChannelInterface::class) + ->onlyMethods(['notice']) + ->getMockForAbstractClass(); + // Expect one notice. + $logger->expects($this->once()) + ->method('notice') + ->with('Cancelling resource processing; resource has changed.'); + $this->container->set('dkan.datastore.logger_channel', $logger); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $this->container->get('database'), + ); + + $result = $post_import->processResource($resource_b); + + $this->assertEquals( + 'Cancelling resource processing; resource has changed.', + $result->getPostImportMessage() + ); + $this->assertEquals( + 'error', + $result->getPostImportStatus() + ); + } + + /** + * @covers ::processResource + */ + public function testProcessResourceNonExistentResource() { + $this->installEntitySchema('resource_mapping'); + + $this->config('metastore.settings') + ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->save(); + + $resource = new DataResource('test.csv', 'text/csv'); + $resource_mapper = $this->getMockBuilder(ResourceMapper::class) + ->disableOriginalConstructor() + ->onlyMethods(['get']) + ->getMock(); + // Resource returns NULL + $resource_mapper->expects($this->once()) + ->method('get') + ->willReturn(NULL); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + // Mock a logger to expect error logging. + $logger = $this->getMockBuilder(LoggerChannelInterface::class) + ->onlyMethods(['notice']) + ->getMockForAbstractClass(); + // Expect one notice. + $logger->expects($this->once()) + ->method('notice') + ->with('Cancelling resource processing; resource no longer exists.'); + $this->container->set('dkan.datastore.logger_channel', $logger); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $this->container->get('database'), + ); + + $result = $post_import->processResource($resource); + + $this->assertEquals( + 'Cancelling resource processing; resource no longer exists.', + $result->getPostImportMessage() + ); + $this->assertEquals( + 'error', + $result->getPostImportStatus() + ); + } + + /** + * @covers ::processResource + */ + public function testProcessResourceErrorWithFailingDrop() { + $this->installEntitySchema('resource_mapping'); + + $this->config('metastore.settings') + ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->save(); + + $this->config('datastore.settings') + ->set('drop_datastore_on_post_import_error', TRUE) + ->save(); + + $resource = new DataResource('test.csv', 'text/csv'); + $resource_mapper = $this->getMockBuilder(ResourceMapper::class) + ->disableOriginalConstructor() + ->onlyMethods(['get']) + ->getMock(); + $resource_mapper->expects($this->once()) + ->method('get') + ->willReturn($resource); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + // Our error result. + $error_result = $this->getMockBuilder(PostImportResult::class) + ->disableOriginalConstructor() + ->onlyMethods(['getPostImportStatus']) + ->getMock(); + $error_result->expects($this->any()) + ->method('getPostImportStatus') + ->willReturn('error'); + + // Mock a logger to expect error logging. + $logger = $this->getMockBuilder(LoggerChannelInterface::class) + ->onlyMethods(['error']) + ->getMockForAbstractClass(); + $logger->expects($this->any()) + ->method('error'); + $this->container->set('dkan.datastore.logger_channel', $logger); + + // Datastore service rigged to explode. + $datastore_service = $this->getMockBuilder(DatastoreService::class) + ->disableOriginalConstructor() + ->onlyMethods(['drop']) + ->getMock(); + $datastore_service->expects($this->once()) + ->method('drop') + ->willThrowException(new \Exception('drop error')); + $this->container->set('dkan.datastore.service', $datastore_service); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $this->container->get('database'), + ); + + $result = $post_import->processResource($resource); + + $this->assertEquals( + 'Attempted to retrieve a sitewide data dictionary, but none was set.', + $result->getPostImportMessage() + ); + $this->assertEquals( + 'error', + $result->getPostImportStatus() + ); + } + + /** + * @covers ::processResource + */ + public function testProcessResourceDropOnPostImportDisabled() { + $this->installEntitySchema('resource_mapping'); + + $this->config('metastore.settings') + ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->save(); + + // Do NOT drop on post import error + $this->config('datastore.settings') + ->set('drop_datastore_on_post_import_error', FALSE) + ->save(); + + $resource = new DataResource('test.csv', 'text/csv'); + $resource_mapper = $this->getMockBuilder(ResourceMapper::class) + ->disableOriginalConstructor() + ->onlyMethods(['get']) + ->getMock(); + $resource_mapper->expects($this->once()) + ->method('get') + ->willReturn($resource); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + // Mock the dictionary enforcer to throw an exception + $enforcer = $this->getMockBuilder(DictionaryEnforcer::class) + ->disableOriginalConstructor() + ->onlyMethods(['process']) + ->getMock(); + $enforcer->expects($this->once()) + ->method('process') + ->willThrowException(new \Exception('our test message')); + $this->container->set('dkan.datastore.service.resource_processor.dictionary_enforcer', $enforcer); + + // Mock a logger to expect error logging. + $logger = $this->getMockBuilder(LoggerChannelInterface::class) + ->onlyMethods(['error']) + ->getMockForAbstractClass(); + $logger->expects($this->once()) + ->method('error') + ->with('our test message'); + $this->container->set('dkan.datastore.logger_channel', $logger); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $this->container->get('database'), + ); + + $result = $post_import->processResource($resource); + + $this->assertEquals( + 'our test message', + $result->getPostImportMessage() + ); + $this->assertEquals( + 'error', + $result->getPostImportStatus() + ); + } + + /** + * @covers ::processResource + */ + public function testProcessResourceErrorWithSuccessfulDrop() { + $this->installEntitySchema('resource_mapping'); + + $this->config('metastore.settings') + ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) + ->save(); + + $this->config('datastore.settings') + ->set('drop_datastore_on_post_import_error', TRUE) + ->save(); + + $resource = new DataResource('test.csv', 'text/csv'); + $resource_mapper = $this->getMockBuilder(ResourceMapper::class) + ->disableOriginalConstructor() + ->onlyMethods(['get']) + ->getMock(); + $resource_mapper->expects($this->once()) + ->method('get') + ->willReturn($resource); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + // Mock the dictionary enforcer to throw an exception + $enforcer = $this->getMockBuilder(DictionaryEnforcer::class) + ->disableOriginalConstructor() + ->onlyMethods(['process']) + ->getMock(); + $enforcer->expects($this->once()) + ->method('process') + ->willThrowException(new \Exception('our test message')); + $this->container->set('dkan.datastore.service.resource_processor.dictionary_enforcer', $enforcer); + + // Mock a logger to expect error logging. + $logger = $this->getMockBuilder(LoggerChannelInterface::class) + ->onlyMethods(['notice']) + ->getMockForAbstractClass(); + $logger->expects($this->once()) + ->method('notice') + ->with( + 'Successfully dropped the datastore for resource @identifier due to a post import error. Visit the Datastore Import Status dashboard for details.', + ['@identifier' => $resource->getIdentifier()], + ); + $this->container->set('dkan.datastore.logger_channel', $logger); + + // Datastore service will always succeed. Mocked so we don't have to deal + // with dropping an actual datastore. + $datastore_service = $this->getMockBuilder(DatastoreService::class) + ->disableOriginalConstructor() + ->onlyMethods(['drop']) + ->getMock(); + $datastore_service->expects($this->once()) + ->method('drop'); + $this->container->set('dkan.datastore.service', $datastore_service); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $this->container->get('database'), + ); + + $result = $post_import->processResource($resource); + + $this->assertEquals( + 'our test message', + $result->getPostImportMessage() + ); + $this->assertEquals( + 'error', + $result->getPostImportStatus() + ); + } + + /** + * @covers ::processResource + */ + public function testProcessResourceDataDictionaryDisabled() { + $this->config('metastore.settings') + ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_NONE) + ->save(); + + $resource = new DataResource('test.csv', 'text/csv'); + $resource_mapper = $this->getMockBuilder(ResourceMapper::class) + ->disableOriginalConstructor() + ->onlyMethods(['get']) + ->getMock(); + $resource_mapper->expects($this->once()) + ->method('get') + ->willReturn($resource); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $this->container->get('database'), + ); + + $result = $post_import->processResource($resource); + + $this->assertEquals( + 'Data-Dictionary Disabled', + $result->getPostImportMessage() + ); + $this->assertEquals( + 'N/A', + $result->getPostImportStatus() + ); + } + + /** + * @covers ::processResource + */ + public function testProcessResourceNoDictionary() { + // Tell the processor to use reference mode for dictionary enforcement. + $this->config('metastore.settings') + ->set('data_dictionary_mode', DataDictionaryDiscoveryInterface::MODE_REFERENCE) + ->save(); + + $resource = new DataResource('test.csv', 'text/csv'); + $resource_mapper = $this->getMockBuilder(ResourceMapper::class) + ->disableOriginalConstructor() + ->onlyMethods(['get']) + ->getMock(); + $resource_mapper->expects($this->once()) + ->method('get') + ->willReturn($resource); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + // Mock the dictionary enforcer to throw an exception so that we can avoid + // node type dependencies. + $no_dictionary_exception = new ResourceDoesNotHaveDictionary('test', 123); + $enforcer = $this->getMockBuilder(DictionaryEnforcer::class) + ->disableOriginalConstructor() + ->onlyMethods(['process']) + ->getMock(); + $enforcer->expects($this->once()) + ->method('process') + ->willThrowException($no_dictionary_exception); + $this->container->set('dkan.datastore.service.resource_processor.dictionary_enforcer', $enforcer); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $this->container->get('database'), + ); + + $result = $post_import->processResource($resource); + + $this->assertEquals( + 'Resource does not have a data dictionary.', + $result->getPostImportMessage() + ); + $this->assertEquals( + 'done', + $result->getPostImportStatus() + ); + } + /** * @covers ::retrieveJobStatus */ @@ -38,8 +472,14 @@ public function testRetrieveJobStatusException() { ->willThrowException(new \Exception()); $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), $connection, - $this->container->get('dkan.metastore.resource_mapper') ); $this->assertFalse($post_import->retrieveJobStatus('id', '123')); @@ -58,12 +498,45 @@ public function testRemoveJobStatusException() { ->method('delete') ->willThrowException(new \Exception()); - $post_import = new PostImport( - $connection, - $this->container->get('dkan.metastore.resource_mapper') - ); + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $connection, + ); - $this->assertFalse($post_import->retrieveJobStatus('id', '123')); + $this->assertFalse($post_import->removeJobStatus('id', '123')); + } + + /** + * @covers ::storeJobStatus + */ + public function testStoreJobStatusException() { + // Mock a connection to explode. + $connection = $this->getMockBuilder(Connection::class) + ->disableOriginalConstructor() + ->onlyMethods(['insert']) + ->getMockForAbstractClass(); + $connection->expects($this->any()) + ->method('insert') + ->willThrowException(new \Exception()); + + $post_import = new PostImport( + $this->container->get('config.factory'), + $this->container->get('dkan.datastore.logger_channel'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service.resource_processor_collector'), + $this->container->get('dkan.metastore.data_dictionary_discovery'), + $this->container->get('dkan.metastore.reference_lookup'), + $this->container->get('dkan.datastore.service'), + $connection, + ); + + $this->assertFalse($post_import->storeJobStatus('id', '123', 'done', 'done')); } } diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/PostImportResourceProcessorTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/PostImportResourceProcessorTest.php deleted file mode 100644 index 3d8551a541..0000000000 --- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/PostImportResourceProcessorTest.php +++ /dev/null @@ -1,358 +0,0 @@ -getMockBuilder(DataDictionaryDiscovery::class) - ->onlyMethods(['getDataDictionaryMode']) - ->disableOriginalConstructor() - ->getMock(); - - $dataDictionaryDiscovery->method('getDataDictionaryMode') - ->willReturn("sitewide"); - - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process') - ->getMock(); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', $resource); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource); - - $this->assertEquals($dataDictionaryDiscovery->getDataDictionaryMode(), DataDictionaryDiscoveryInterface::MODE_SITEWIDE); - $this->assertEquals($resource->getIdentifier(), $postImportResult->getResourceIdentifier()); - $this->assertEquals($resource->getVersion(), $postImportResult->getResourceVersion()); - $this->assertEquals('done', $postImportResult->getPostImportStatus()); - $this->assertEquals(NULL, $postImportResult->getPostImportMessage()); - - // Ensure resources were processed. - $notices = $container_chain->getStoredInput('notice'); - $this->assertContains('Post import job for resource @id completed.', $notices); - // Ensure no exceptions were thrown. - $errors = $container_chain->getStoredInput('error'); - $this->assertEmpty($errors); - } - - /** - * Test postImportProcessItem() DataDictionary disabled. - * - * @covers ::postImportProcessItem - */ - public function testPostImportProcessItemDataDictionaryDisabled() { - $resource = new DataResource('test.csv', 'text/csv'); - - $dataDictionaryDiscovery = $this->getMockBuilder(DataDictionaryDiscovery::class) - ->onlyMethods(['getDataDictionaryMode']) - ->disableOriginalConstructor() - ->getMock(); - - $dataDictionaryDiscovery->method('getDataDictionaryMode') - ->willReturn("none"); - - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process') - ->getMock(); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', $resource) - ->add(DataDictionaryDiscoveryInterface::class, 'getDataDictionaryMode', 'none') - ->add(DataDictionaryDiscovery::class, 'getDataDictionaryMode', 'none'); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource); - - $this->assertEquals($dataDictionaryDiscovery->getDataDictionaryMode(), DataDictionaryDiscoveryInterface::MODE_NONE); - $this->assertEquals($resource->getIdentifier(), $postImportResult->getResourceIdentifier()); - $this->assertEquals($resource->getVersion(), $postImportResult->getResourceVersion()); - $this->assertEquals('N/A', $postImportResult->getPostImportStatus()); - $this->assertEquals('Data-Dictionary Disabled', $postImportResult->getPostImportMessage()); - } - - /** - * Test postImportProcessItem() halts and logs a message if a resource no longer exists. - * - * @covers ::postImportProcessItem - */ - public function testPostImportProcessItemResourceNoLongerExists() { - $resource = new DataResource('test.csv', 'text/csv'); - - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process') - ->getMock(); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', NULL); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource); - - $this->assertEquals($resource->getIdentifier(), $postImportResult->getResourceIdentifier()); - $this->assertEquals($resource->getVersion(), $postImportResult->getResourceVersion()); - $this->assertEquals('error', $postImportResult->getPostImportStatus()); - $this->assertEquals('Cancelling resource processing; resource no longer exists.', $postImportResult->getPostImportMessage()); - - // Ensure notice was logged and resource processing was halted. - $notices = $container_chain->getStoredInput('notice'); - $this->assertEquals($notices[0], 'Cancelling resource processing; resource no longer exists.'); - // Ensure no exceptions were thrown. - $errors = $container_chain->getStoredInput('error'); - $this->assertEmpty($errors); - } - - /** - * Test postImportProcessItem() halts, logs message if resource has changed. - * - * @covers ::postImportProcessItem - */ - public function testPostImportProcessItemResourceChanged() { - $resource_a = new DataResource('test.csv', 'text/csv'); - - $resource_b = (new DataResource('test2.csv', 'text/csv'))->createNewVersion(); - - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process') - ->getMock(); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', $resource_a); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource_b); - - $this->assertEquals($resource_b->getIdentifier(), $postImportResult->getResourceIdentifier()); - $this->assertEquals($resource_b->getVersion(), $postImportResult->getResourceVersion()); - $this->assertEquals('error', $postImportResult->getPostImportStatus()); - $this->assertEquals('Cancelling resource processing; resource has changed.', $postImportResult->getPostImportMessage()); - - // Ensure notice was logged and resource processing was halted. - $notices = $container_chain->getStoredInput('notice'); - $this->assertEquals($notices[0], 'Cancelling resource processing; resource has changed.'); - // Ensure no exceptions were thrown. - $errors = $container_chain->getStoredInput('error'); - $this->assertEmpty($errors); - } - - /** - * Test postImportProcessItem() logs errors encountered in processors. - * - * @covers ::postImportProcessItem - */ - public function testPostImportProcessItemProcessorError() { - $resource = new DataResource('test.csv', 'text/csv'); - - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process', new \Exception('Test Error')) - ->getMock(); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', $resource); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource); - - $this->assertEquals($resource->getIdentifier(), $postImportResult->getResourceIdentifier()); - $this->assertEquals($resource->getVersion(), $postImportResult->getResourceVersion()); - $this->assertEquals('error', $postImportResult->getPostImportStatus()); - $this->assertEquals('Test Error', $postImportResult->getPostImportMessage()); - - // Ensure resources were processed. - $notices = $container_chain->getStoredInput('notice'); - $this->assertEmpty($notices); - // Ensure test error was caught. - $errors = $container_chain->getStoredInput('error'); - $this->assertEquals($errors[0], 'Test Error'); - } - - /** - * Verify Datastore Drop on Post-Import Error (with drop_config enabled) - * - * @covers ::postImportProcessItem - */ - public function testDatastoreDropOnPostImportError() { - $resource = new DataResource('test.csv', 'text/csv'); - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process', new \Exception('Test Error')) - ->getMock(); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', $resource); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource); - - $this->assertEquals($resource->getIdentifier(), $postImportResult->getResourceIdentifier()); - $this->assertEquals($resource->getVersion(), $postImportResult->getResourceVersion()); - $this->assertEquals('error', $postImportResult->getPostImportStatus()); - $this->assertEquals('Test Error', $postImportResult->getPostImportMessage()); - } - - /** - * Verify Logging on Successful Datastore Drop. - * - * @covers ::postImportProcessItem - */ - public function testLoggingOnSuccessfulDatastoreDrop() { - $resource = new DataResource('test.csv', 'text/csv'); - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process') - ->getMock(); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', $resource); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource); - - $this->assertEquals($resource->getIdentifier(), $postImportResult->getResourceIdentifier()); - $this->assertEquals($resource->getVersion(), $postImportResult->getResourceVersion()); - } - - /** - * Verify No Datastore Drop When drop_config is Disabled. - * - * @covers ::postImportProcessItem - */ - public function testNoDatastoreDropWhenDropConfigIsDisabled() { - $resource = new DataResource('test.csv', 'text/csv'); - $resource_processor = (new Chain($this)) - ->add(ResourceProcessorInterface::class, 'process') - ->getMock(); - - $datastoreService = $this->createMock(DatastoreService::class); - $datastoreService->expects($this->never()) - ->method('drop'); - - $container_chain = $this->getContainerChain() - ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$resource_processor]) - ->add(ResourceMapper::class, 'get', $resource); - \Drupal::setContainer($container_chain->getMock()); - - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); - - $postImportResult = $dictionaryEnforcer->postImportProcessItem($resource); - - $this->assertEquals('done', $postImportResult->getPostImportStatus()); - $this->assertEquals(NULL, $postImportResult->getPostImportMessage()); - } - - /** - * Get container chain. - */ - protected function getContainerChain() { - - $options = (new Options()) - ->add('config.factory', ConfigFactoryInterface::class) - ->add('dkan.datastore.data_dictionary.alter_table_query_builder.mysql', AlterTableQueryBuilderInterface::class) - ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscovery::class) - ->add('dkan.datastore.logger_channel', LoggerInterface::class) - ->add('dkan.metastore.service', MetastoreService::class) - ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscoveryInterface::class) - ->add('stream_wrapper_manager', StreamWrapperManager::class) - ->add('dkan.metastore.resource_mapper', ResourceMapper::class) - ->add('dkan.datastore.service', DatastoreService::class) - ->add('dkan.datastore.service.resource_processor_collector', ResourceProcessorCollector::class) - ->add('dkan.datastore.service.post_import', PostImport::class) - ->add('dkan.metastore.reference_lookup', ReferenceLookup::class) - ->index(0); - - $json = '{"identifier":"foo","title":"bar","data":{"fields":[]}}'; - - return (new Chain($this)) - ->add(Container::class, 'get', $options) - ->add(LoggerInterface::class, 'error', NULL, 'error') - ->add(LoggerInterface::class, 'notice', '', 'notice') - ->add(MetastoreService::class, 'get', new RootedJsonData($json)) - ->add(AlterTableQueryBuilderInterface::class, 'setConnectionTimeout', AlterTableQueryBuilderInterface::class) - ->add(AlterTableQueryBuilderInterface::class, 'getQuery', AlterTableQueryInterface::class) - ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', 'resource_id') - ->add(PublicStream::class, 'getExternalUrl', self::HOST) - ->add(StreamWrapperManager::class, 'getViaUri', PublicStream::class) - ->add(ResourceMapper::class, 'get', DataResource::class) - ->add(ConfigFactoryInterface::class, 'get', FALSE) - ->add(DatastoreService::class, 'drop'); - } - -} diff --git a/modules/datastore/tests/src/Unit/Service/PostImportTest.php b/modules/datastore/tests/src/Unit/Service/PostImportTest.php index 6281387e10..5a7e2237fb 100644 --- a/modules/datastore/tests/src/Unit/Service/PostImportTest.php +++ b/modules/datastore/tests/src/Unit/Service/PostImportTest.php @@ -6,6 +6,13 @@ use Drupal\metastore\ResourceMapper; use Drupal\datastore\Service\PostImport; use PHPUnit\Framework\TestCase; +use Psr\Log\LoggerInterface; +use Drupal\Core\Config\ConfigFactoryInterface; +use Drupal\datastore\Service\ResourceProcessorCollector; +use Drupal\metastore\DataDictionary\DataDictionaryDiscoveryInterface; +use Drupal\metastore\Reference\ReferenceLookup; +use Drupal\datastore\DatastoreService; +use Drupal\common\DataResource; /** * Tests the PostImport service. @@ -17,68 +24,165 @@ class PostImportTest extends TestCase { /** - * The resource mapper mock. + * Test storeJobStatus() succeeds. * - * @var \Drupal\metastore\ResourceMapper - */ - protected $resourceMapper; + * @covers ::storeJobStatus + */ + public function testStoreJobStatus() { + $mocks = $this->getMockDependencies(); + + $queryMock = $this->getMockBuilder('stdClass') + ->addMethods(['fields', 'execute']) + ->getMock(); + + $queryMock->expects($this->once()) + ->method('fields') + ->with([ + 'resource_identifier' => 'test_identifier', + 'resource_version' => 'test_version', + 'post_import_status' => 'test_status', + 'post_import_error' => 'test_error', + ]) + ->willReturnSelf(); + + $queryMock->expects($this->once()) + ->method('execute') + ->willReturn(TRUE); + + $mocks['connection']->expects($this->once()) + ->method('insert') + ->with('dkan_post_import_job_status') + ->willReturn($queryMock); + + $post_import = new PostImport( + ...array_values($mocks), + ); + + $result_store = $post_import->storeJobStatus('test_identifier', 'test_version', 'test_status', 'test_error'); + + $this->assertTrue($result_store); + } /** - * The database connection. + * Test retrieveJobStatus() succeeds. * - * @var \Drupal\Core\Database\Connection - */ - protected $connectionMock; + * @covers ::retrieveJobStatus + */ + public function testRetrieveJobStatus() { + $import_info = [ + '#resource_version' => 'test_version', + '#post_import_status' => 'test_status', + '#post_import_error' => 'test_error', + ]; + + $mocks = $this->getMockDependencies(); + + $resultMock = $this->getMockBuilder('stdClass') + ->addMethods(['fetchAssoc']) + ->getMock(); + + $resultMock->expects($this->once()) + ->method('fetchAssoc') + ->willReturn($import_info); + + $queryMock = $this->getMockBuilder('stdClass') + ->addMethods(['condition', 'fields', 'execute']) + ->getMock(); + + $queryMock->expects($this->exactly(2)) + ->method('condition') + ->willReturnSelf(); + + $queryMock->expects($this->once()) + ->method('fields') + ->with('dkan_post_import_job_status', [ + 'resource_version', + 'post_import_status', + 'post_import_error', + ]) + ->willReturnSelf(); + + $queryMock->expects($this->once()) + ->method('execute') + ->willReturn($resultMock); + + $mocks['connection']->expects($this->once()) + ->method('select') + ->with('dkan_post_import_job_status') + ->willReturn($queryMock); + + $post_import = new PostImport( + ...array_values($mocks), + ); + + $result_store = $post_import->retrieveJobStatus('test_identifier', 'test_version'); + + $this->assertSame($result_store, $import_info); + } /** - * The PostImport service. + * Test removeJobStatus() succeeds. * - * @var \Drupal\datastore\Service\PostImport - */ - protected $postImport; + * @covers ::removeJobStatus + */ + public function testRemoveJobStatus() { + $mocks = $this->getMockDependencies(); - /** - * Tests the storeJobStatus function. - */ - public function testStoreJobStatus() { + $resourceMock = $this->getMockBuilder(DataResource::class) + ->disableOriginalConstructor() + ->onlyMethods(['getVersion']) + ->getMock(); - $connectionMock = $this->getMockBuilder(Connection::class) - ->disableOriginalConstructor() - ->getMock(); + $resourceMock->expects($this->once()) + ->method('getVersion') + ->willReturn('test_version'); - $resourceMapperMock = $this->getMockBuilder(ResourceMapper::class) - ->disableOriginalConstructor() - ->getMock(); + $mocks['resourceMapper']->expects($this->once()) + ->method('get') + ->with('test_identifier') + ->willReturn($resourceMock); - $queryMock = $this->getMockBuilder('stdClass') - ->addMethods(['fields', 'execute']) - ->getMock(); + $queryMock = $this->getMockBuilder('stdClass') + ->addMethods(['condition', 'execute']) + ->getMock(); - $connectionMock->expects($this->once()) - ->method('insert') - ->with('dkan_post_import_job_status') - ->willReturn($queryMock); + $queryMock->expects($this->exactly(2)) + ->method('condition') + ->willReturnSelf(); - $queryMock->expects($this->once()) - ->method('fields') - ->with([ - 'resource_identifier' => 'test_identifier', - 'resource_version' => 'test_version', - 'post_import_status' => 'test_status', - 'post_import_error' => 'test_error', - ]) - ->willReturnSelf(); + $queryMock->expects($this->once()) + ->method('execute') + ->willReturn(TRUE); - $queryMock->expects($this->once()) - ->method('execute') - ->willReturn(TRUE); + $mocks['connection']->expects($this->once()) + ->method('delete') + ->with('dkan_post_import_job_status') + ->willReturn($queryMock); - $postImport = new PostImport($connectionMock, $resourceMapperMock); + $post_import = new PostImport( + ...array_values($mocks), + ); - $result_store = $postImport->storeJobStatus('test_identifier', 'test_version', 'test_status', 'test_error'); + $result_store = $post_import->removeJobStatus('test_identifier'); - // Assert that the method returned the expected result. - $this->assertTrue($result_store); + $this->assertTrue($result_store); } + /** + * Setup PostImport container mocks. + */ + public function getMockDependencies() { + return [ + 'configFactory' => $this->createMock(ConfigFactoryInterface::class), + 'logger' => $this->createMock(LoggerInterface::class), + 'resourceMapper' => $this->createMock(ResourceMapper::class), + 'resourceProcessorCollector' => $this->createMock(ResourceProcessorCollector::class), + 'dataDictionaryDiscovery' => $this->createMock(DataDictionaryDiscoveryInterface::class), + 'referenceLookup' => $this->createMock(ReferenceLookup::class), + 'datastoreService' => $this->createMock(DatastoreService::class), + 'connection' => $this->createMock(Connection::class), + ]; + } + + } diff --git a/modules/datastore/tests/src/Unit/Service/ResourceProcessor/DictionaryEnforcerTest.php b/modules/datastore/tests/src/Unit/Service/ResourceProcessor/DictionaryEnforcerTest.php index 84c9d2d418..d130908332 100644 --- a/modules/datastore/tests/src/Unit/Service/ResourceProcessor/DictionaryEnforcerTest.php +++ b/modules/datastore/tests/src/Unit/Service/ResourceProcessor/DictionaryEnforcerTest.php @@ -3,6 +3,7 @@ namespace Drupal\Tests\datastore\Unit\Service\ResourceProcessor; use Drupal\Core\Config\ConfigFactoryInterface; +use Drupal\Core\Config\Config; use Drupal\Core\DependencyInjection\Container; use Drupal\Core\StreamWrapper\PublicStream; use Drupal\Core\StreamWrapper\StreamWrapperManager; @@ -24,6 +25,7 @@ use PHPUnit\Framework\TestCase; use Psr\Log\LoggerInterface; use RootedData\RootedJsonData; +use Drupal\Core\Database\Connection; /** * Test \Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer. @@ -50,15 +52,19 @@ public function testProcess() { $resource = new DataResource('test.csv', 'text/csv'); $alter_table_query_builder = (new Chain($this)) - ->add(AlterTableQueryBuilderInterface::class, 'getQuery', AlterTableQueryInterface::class) - ->add(AlterTableQueryInterface::class, 'execute') - ->getMock(); + ->add(AlterTableQueryBuilderInterface::class, 'getQuery', AlterTableQueryInterface::class) + ->add(AlterTableQueryInterface::class, 'execute') + ->getMock(); + $metastore_service = (new Chain($this)) ->add(MetastoreService::class, 'get', new RootedJsonData(json_encode(['data' => ['fields' => []]]))) ->getMock(); + $dictionary_discovery_service = (new Chain($this)) ->add(DataDictionaryDiscoveryInterface::class, 'dictionaryIdFromResource', 'dictionary-id') + ->add(DataDictionaryDiscoveryInterface::class, 'getDataDictionaryMode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE) ->getMock(); + $dictionary_enforcer = new DictionaryEnforcer($alter_table_query_builder, $metastore_service, $dictionary_discovery_service); $container_chain = $this->getContainerChain($resource->getVersion()) @@ -67,15 +73,18 @@ public function testProcess() { ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$dictionary_enforcer]); \Drupal::setContainer($container_chain->getMock($resource->getVersion())); - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] + // Test with no errors + $mocks = $this->getMockDependencies($resource, '', $dictionary_enforcer); + + $post_import = new PostImport( + ...array_values($mocks), ); - $dictionaryEnforcer->postImportProcessItem($resource); + $post_import_resource_processor = new PostImportResourceProcessor( + [], '', ['cron' => ['lease_time' => 10800]], $post_import, $this->createMock(ConfigFactoryInterface::class) + ); - // Assert no exceptions are thrown. - $errors = $container_chain->getStoredInput('error'); - $this->assertEmpty($errors); + $post_import_resource_processor->processItem($resource); } /** @@ -104,15 +113,18 @@ public function testProcessItemExecuteException() { ->add(ResourceProcessorCollector::class, 'getResourceProcessors', [$dictionary_enforcer]); \Drupal::setContainer($container_chain->getMock($resource->getVersion())); - $dictionaryEnforcer = PostImportResourceProcessor::create( - $container_chain->getMock(), [], '', ['cron' => ['lease_time' => 10800]] - ); + // Test with error + $mocks = $this->getMockDependencies($resource, 'error', $dictionary_enforcer); - $dictionaryEnforcer->postImportProcessItem($resource); + $post_import = new PostImport( + ...array_values($mocks), + ); - // Assert no exceptions are thrown. - $errors = $container_chain->getStoredInput('error'); - $this->assertEquals($errors[0], 'Test Error'); + $post_import_resource_processor = new PostImportResourceProcessor( + [], '', ['cron' => ['lease_time' => 10800]], $post_import, $this->createMock(ConfigFactoryInterface::class) + ); + + $post_import_resource_processor->processItem($resource); } /** @@ -146,6 +158,70 @@ public function testReturnDataDictionaryFields() { $this->assertIsArray($result); } + /** + * Setup PostImport container mocks. + */ + public function getMockDependencies($resource, $expectation, $dictionary_enforcer) { + $resourceMapperMock = $this->createMock(ResourceMapper::class); + $resourceMapperMock->expects($this->any()) + ->method('get') + ->withAnyParameters() + ->willReturn($resource); + + $configMock = $this->createMock(Config::class); + $configFactoryMock = $this->createMock(ConfigFactoryInterface::class); + + $configFactoryMock + ->method('get') + ->with('datastore.settings') + ->willReturn($configMock); + + $configMock + ->method('get') + ->with('drop_datastore_on_post_import_error') + ->willReturn(false); + + $resourceProcessorMock = $this->createMock(ResourceProcessorCollector::class); + $resourceProcessorMock->expects($this->any()) + ->method('getResourceProcessors') + ->willReturn([$dictionary_enforcer]); + + $queryMock = $this->getMockBuilder('stdClass') + ->addMethods(['fields', 'execute']) + ->getMock(); + + $queryMock->expects($this->once()) + ->method('fields') + ->with([ + 'resource_identifier' => $resource->getIdentifier(), + 'resource_version' => $resource->getVersion(), + 'post_import_status' => ($expectation === "error") ? 'error' : 'done', + 'post_import_error' => ($expectation === "error") ? 'Test Error' : '', + ]) + ->willReturnSelf(); + + $queryMock->expects($this->once()) + ->method('execute') + ->willReturn(TRUE); + + $connectionMock = $this->createMock(Connection::class); + $connectionMock ->expects($this->once()) + ->method('insert') + ->with('dkan_post_import_job_status') + ->willReturn($queryMock); + + return [ + 'configFactory' => $configFactoryMock , + 'logger' => $this->createMock(LoggerInterface::class), + 'resourceMapper' => $resourceMapperMock, + 'resourceProcessorCollector' => $resourceProcessorMock, + 'dataDictionaryDiscovery' => $this->createMock(DataDictionaryDiscoveryInterface::class), + 'referenceLookup' => $this->createMock(ReferenceLookup::class), + 'datastoreService' => $this->createMock(DatastoreService::class), + 'connection' => $connectionMock, + ]; + } + /** * Get container chain. */ @@ -155,6 +231,7 @@ protected function getContainerChain(int $resource_version) { ->add('config.factory', ConfigFactoryInterface::class) ->add('dkan.datastore.data_dictionary.alter_table_query_builder.mysql', AlterTableQueryBuilderInterface::class) ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscovery::class) + ->add('dkan.datastore.service.post_import', PostImport::class) ->add('dkan.datastore.logger_channel', LoggerInterface::class) ->add('dkan.metastore.service', MetastoreService::class) ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscoveryInterface::class) @@ -163,7 +240,6 @@ protected function getContainerChain(int $resource_version) { ->add('dkan.datastore.service', DatastoreService::class) ->add('dkan.datastore.service.resource_processor_collector', ResourceProcessorCollector::class) ->add('dkan.datastore.service.resource_processor.dictionary_enforcer', DictionaryEnforcer::class) - ->add('dkan.datastore.service.post_import', PostImport::class) ->add('dkan.metastore.reference_lookup', ReferenceLookup::class) ->index(0);