From 496976a487d5338c1b414c037d3a107a3cc3480b Mon Sep 17 00:00:00 2001 From: Paul Mitchum Date: Wed, 22 Nov 2023 16:31:42 -0600 Subject: [PATCH] ImportInfo and ImportJob stores out of sync (#4050) --- composer.json | 4 +- .../src/Functional/FileFetcherFactoryTest.php | 2 + modules/datastore/datastore.services.yml | 4 +- .../src/Factory/MysqlImportFactory.php | 27 +-- .../src/Service/MysqlImport.php | 5 +- .../src/Controller/ImportController.php | 37 +++- modules/datastore/src/DatastoreResource.php | 4 + modules/datastore/src/DatastoreService.php | 33 ++-- .../src/Plugin/QueueWorker/ImportJob.php | 10 +- .../Service/Factory/ImportServiceFactory.php | 7 +- .../datastore/src/Service/ImportService.php | 57 +++--- .../datastore/src/Service/Info/ImportInfo.php | 171 ++++++++++-------- .../src/Service/Info/ImportInfoList.php | 6 +- .../Controller/ImportControllerTest.php | 39 ++++ .../QueueWorker/ImportQueueWorkerTest.php | 3 - .../src/Kernel/Service/ImportServiceTest.php | 106 +++++++++++ .../Service/Info/ImportInfoListTest.php | 82 +++++++++ .../Kernel/Service/Info/ImportInfoTest.php | 130 +++++++++++++ .../Unit/Controller/ImportControllerTest.php | 63 ------- .../tests/src/Unit/DatastoreServiceTest.php | 4 +- .../Unit/Plugin/QueueWorker/ImportJobTest.php | 7 +- .../src/Unit/Service/ImportServiceTest.php | 162 ----------------- .../Unit/Service/Info/ImportInfoListTest.php | 48 ----- .../Unit/Service/ResourceLocalizerTest.php | 1 - 24 files changed, 560 insertions(+), 452 deletions(-) create mode 100644 modules/datastore/tests/src/Kernel/Controller/ImportControllerTest.php create mode 100644 modules/datastore/tests/src/Kernel/Service/ImportServiceTest.php create mode 100644 modules/datastore/tests/src/Kernel/Service/Info/ImportInfoListTest.php create mode 100644 modules/datastore/tests/src/Kernel/Service/Info/ImportInfoTest.php delete mode 100644 modules/datastore/tests/src/Unit/Controller/ImportControllerTest.php delete mode 100644 modules/datastore/tests/src/Unit/Service/ImportServiceTest.php delete mode 100644 modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php diff --git a/composer.json b/composer.json index d93fa6581e..029784a498 100644 --- a/composer.json +++ b/composer.json @@ -15,9 +15,9 @@ "fmizzell/maquina": "^1.1.1", "getdkan/contracts": "^1.0.0", "getdkan/csv-parser": "^1.3.0", - "getdkan/file-fetcher" : "^4.2.2", + "getdkan/file-fetcher" : "^5.0.0", "getdkan/harvest": "^1.0.0", - "getdkan/procrastinator": "^4.0.0", + "getdkan/procrastinator": "^5.0.0", "getdkan/rooted-json-data": "^0.1", "guzzlehttp/guzzle" : "^6.5.8 || ^7.4.5", "ilbee/csv-response": "^1.1.1", diff --git a/modules/common/tests/src/Functional/FileFetcherFactoryTest.php b/modules/common/tests/src/Functional/FileFetcherFactoryTest.php index 7e60d38b69..19cae4bf37 100644 --- a/modules/common/tests/src/Functional/FileFetcherFactoryTest.php +++ b/modules/common/tests/src/Functional/FileFetcherFactoryTest.php @@ -2,6 +2,7 @@ namespace Drupal\Tests\common\Kernel\FileFetcher; +use Drupal\common\FileFetcher\DkanFileFetcher; use Drupal\common\FileFetcher\FileFetcherFactory; use Drupal\KernelTests\KernelTestBase; use FileFetcher\FileFetcher; @@ -63,6 +64,7 @@ public function testOurRemote($use_existing, $remote_class) { ]; $ff = $factory->getInstance('identifier', $config); $this->assertInstanceOf(FileFetcher::class, $ff); + $this->assertInstanceOf(DkanFileFetcher::class, $ff); // Make sure we have the correct processor class that corresponds to our // config. diff --git a/modules/datastore/datastore.services.yml b/modules/datastore/datastore.services.yml index 5828856649..89677b6c0b 100644 --- a/modules/datastore/datastore.services.yml +++ b/modules/datastore/datastore.services.yml @@ -6,7 +6,6 @@ services: - '@dkan.datastore.service.factory.import' - '@queue' - '@dkan.common.job_store' - - '@dkan.datastore.import_info_list' - '@dkan.datastore.service.resource_processor.dictionary_enforcer' dkan.datastore.service.post_import: @@ -101,9 +100,10 @@ services: dkan.datastore.import_info: class: \Drupal\datastore\Service\Info\ImportInfo arguments: - - '@dkan.common.job_store' - '@dkan.datastore.service.resource_localizer' - '@dkan.datastore.service.factory.import' + - '@dkan.metastore.resource_mapper' + - '@dkan.datastore.service' dkan.datastore.import_info_list: class: \Drupal\datastore\Service\Info\ImportInfoList diff --git a/modules/datastore/modules/datastore_mysql_import/src/Factory/MysqlImportFactory.php b/modules/datastore/modules/datastore_mysql_import/src/Factory/MysqlImportFactory.php index 9916ec3d98..3a0a8280d9 100644 --- a/modules/datastore/modules/datastore_mysql_import/src/Factory/MysqlImportFactory.php +++ b/modules/datastore/modules/datastore_mysql_import/src/Factory/MysqlImportFactory.php @@ -9,9 +9,7 @@ use Drupal\datastore_mysql_import\Storage\MySqlDatabaseTableFactory; /** - * Importer factory. - * - * @codeCoverageIgnore + * Mysql importer factory. */ class MysqlImportFactory implements ImportFactoryInterface { @@ -29,13 +27,6 @@ class MysqlImportFactory implements ImportFactoryInterface { */ private $databaseTableFactory; - /** - * Services array. Not really needed, following FactoryInterface. - * - * @var array - */ - private $services = []; - /** * Constructor. */ @@ -50,20 +41,14 @@ public function __construct(JobStoreFactory $jobStoreFactory, MySqlDatabaseTable * @inheritdoc */ public function getInstance(string $identifier, array $config = []) { - - if (!isset($config['resource'])) { + $resource = $config['resource'] ?? FALSE; + if (!$resource) { throw new \Exception("config['resource'] is required"); } - $resource = $config['resource']; - - if (!isset($this->services[$identifier])) { - $this->services[$identifier] = new ImportService($resource, $this->jobStoreFactory, $this->databaseTableFactory); - } - - $this->services[$identifier]->setImporterClass(MysqlImport::class); - - return $this->services[$identifier]; + $importer = new ImportService($resource, $this->jobStoreFactory, $this->databaseTableFactory); + $importer->setImporterClass(MysqlImport::class); + return $importer; } } diff --git a/modules/datastore/modules/datastore_mysql_import/src/Service/MysqlImport.php b/modules/datastore/modules/datastore_mysql_import/src/Service/MysqlImport.php index 4496331d8d..78491cb3f2 100644 --- a/modules/datastore/modules/datastore_mysql_import/src/Service/MysqlImport.php +++ b/modules/datastore/modules/datastore_mysql_import/src/Service/MysqlImport.php @@ -41,6 +41,7 @@ class MysqlImport extends ImportJob { * Configuration options. */ protected function __construct(string $identifier, $storage, array $config = NULL) { + // Ensure we can check if the data has already been imported. if (!($config['storage'] instanceof ImportedItemInterface)) { throw new \Exception('Storage must be an instance of ' . ImportedItemInterface::class); } @@ -62,7 +63,7 @@ protected function runIt() { // If the storage table already exists, we already performed an import and // can stop here. if ($this->dataStorage->hasBeenImported()) { - $this->getResult()->setStatus(Result::DONE); + $this->setStatus(Result::DONE); return NULL; } @@ -101,7 +102,7 @@ protected function runIt() { Database::setActiveConnection(); - $this->getResult()->setStatus(Result::DONE); + $this->setStatus(Result::DONE); return NULL; } diff --git a/modules/datastore/src/Controller/ImportController.php b/modules/datastore/src/Controller/ImportController.php index 96454a0eb4..354448be57 100644 --- a/modules/datastore/src/Controller/ImportController.php +++ b/modules/datastore/src/Controller/ImportController.php @@ -3,13 +3,14 @@ namespace Drupal\datastore\Controller; use Drupal\common\DataResource; -use Symfony\Component\DependencyInjection\ContainerInterface; -use Drupal\Core\DependencyInjection\ContainerInjectionInterface; use Drupal\common\JsonResponseTrait; use Drupal\Component\Uuid\Uuid; +use Drupal\Core\DependencyInjection\ContainerInjectionInterface; use Drupal\datastore\DatastoreService; +use Drupal\datastore\Service\Info\ImportInfoList; use Drupal\metastore\MetastoreApiResponse; use Drupal\metastore\Reference\ReferenceLookup; +use Symfony\Component\DependencyInjection\ContainerInterface; use Symfony\Component\HttpFoundation\Request; /** @@ -27,7 +28,28 @@ class ImportController implements ContainerInjectionInterface { * * @var \Drupal\datastore\DatastoreService */ - protected $datastoreService; + protected DatastoreService $datastoreService; + + /** + * Metastore API response service. + * + * @var \Drupal\metastore\MetastoreApiResponse + */ + protected MetastoreApiResponse $metastoreApiResponse; + + /** + * Reference lookup service. + * + * @var \Drupal\metastore\Reference\ReferenceLookup + */ + protected ReferenceLookup $referenceLookup; + + /** + * Import info list service. + * + * @var \Drupal\datastore\Service\Info\ImportInfoList + */ + protected ImportInfoList $importInfoList; /** * Api constructor. @@ -35,11 +57,13 @@ class ImportController implements ContainerInjectionInterface { public function __construct( DatastoreService $datastoreService, MetastoreApiResponse $metastoreApiResponse, - ReferenceLookup $referenceLookup + ReferenceLookup $referenceLookup, + ImportInfoList $importInfoList ) { $this->datastoreService = $datastoreService; $this->metastoreApiResponse = $metastoreApiResponse; $this->referenceLookup = $referenceLookup; + $this->importInfoList = $importInfoList; } /** @@ -49,7 +73,8 @@ public static function create(ContainerInterface $container) { return new ImportController( $container->get('dkan.datastore.service'), $container->get('dkan.metastore.api_response'), - $container->get('dkan.metastore.reference_lookup') + $container->get('dkan.metastore.reference_lookup'), + $container->get('dkan.datastore.import_info_list') ); } @@ -211,7 +236,7 @@ public function deleteMultiple(Request $request) { */ public function list(Request $request) { try { - $data = $this->datastoreService->list(); + $data = $this->importInfoList->buildList(); return $this->metastoreApiResponse->cachedJsonResponse($data, 200, ['distribution'], $request->query); } catch (\Exception $e) { diff --git a/modules/datastore/src/DatastoreResource.php b/modules/datastore/src/DatastoreResource.php index fc04b9c20c..171d0b049e 100644 --- a/modules/datastore/src/DatastoreResource.php +++ b/modules/datastore/src/DatastoreResource.php @@ -4,6 +4,10 @@ /** * Basic datastore resource class. + * + * Always generate this object using DataResource::getDatastoreResource(). + * + * @see \Drupal\common\DataResource::getDatastoreResource() */ class DatastoreResource implements \JsonSerializable { diff --git a/modules/datastore/src/DatastoreService.php b/modules/datastore/src/DatastoreService.php index 206ce03e84..625fa9e533 100644 --- a/modules/datastore/src/DatastoreService.php +++ b/modules/datastore/src/DatastoreService.php @@ -4,16 +4,16 @@ use Drupal\common\DataResource; use Drupal\common\Storage\JobStoreFactory; +use Drupal\datastore\Service\ImportService; use Procrastinator\Result; use Symfony\Component\DependencyInjection\ContainerInterface; use Drupal\Core\DependencyInjection\ContainerInjectionInterface; use Drupal\Core\Queue\QueueFactory; use Drupal\datastore\Plugin\QueueWorker\ImportJob; -use Drupal\datastore\Service\ResourceLocalizer; use Drupal\datastore\Service\Factory\ImportFactoryInterface; -use Drupal\datastore\Service\Info\ImportInfoList; -use FileFetcher\FileFetcher; +use Drupal\datastore\Service\ResourceLocalizer; use Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer; +use FileFetcher\FileFetcher; /** * Main services for the datastore. @@ -30,7 +30,7 @@ class DatastoreService implements ContainerInjectionInterface { /** * Datastore import factory class. * - * @var \Drupal\datastore\Service\Factory\ImportFactoryInterface + * @var \Drupal\datastore\Service\Factory\ImportServiceFactory */ private $importServiceFactory; @@ -64,7 +64,6 @@ public static function create(ContainerInterface $container) { $container->get('dkan.datastore.service.factory.import'), $container->get('queue'), $container->get('dkan.common.job_store'), - $container->get('dkan.datastore.import_info_list'), $container->get('dkan.datastore.service.resource_processor.dictionary_enforcer') ); } @@ -80,8 +79,6 @@ public static function create(ContainerInterface $container) { * Queue factory service. * @param \Drupal\common\Storage\JobStoreFactory $jobStoreFactory * Jobstore factory service. - * @param \Drupal\datastore\Service\Info\ImportInfoList $importInfoList - * Import info list service. * @param \Drupal\datastore\Service\ResourceProcessor\DictionaryEnforcer $dictionaryEnforcer * Dictionary Enforcer object. */ @@ -90,14 +87,12 @@ public function __construct( ImportFactoryInterface $importServiceFactory, QueueFactory $queue, JobStoreFactory $jobStoreFactory, - ImportInfoList $importInfoList, DictionaryEnforcer $dictionaryEnforcer ) { $this->queue = $queue; $this->resourceLocalizer = $resourceLocalizer; $this->importServiceFactory = $importServiceFactory; $this->jobStoreFactory = $jobStoreFactory; - $this->importInfoList = $importInfoList; $this->dictionaryEnforcer = $dictionaryEnforcer; } @@ -150,7 +145,9 @@ public function import(string $identifier, bool $deferred = FALSE, $version = NU private function doImport($resource) { $importService = $this->getImportService($resource); $importService->import(); - return [$this->getLabelFromObject($importService) => $importService->getResult()]; + return [ + $this->getLabelFromObject($importService) => $importService->getImporter()->getResult(), + ]; } /** @@ -197,8 +194,10 @@ private function getResource($identifier, $version) { /** * Getter. */ - public function getImportService(DataResource $resource) { - return $this->importServiceFactory->getInstance($resource->getUniqueIdentifier(), ['resource' => $resource]); + public function getImportService(DataResource $resource): ImportService { + return $this->importServiceFactory->getInstance( + $resource->getUniqueIdentifier(), ['resource' => $resource] + ); } /** @@ -237,16 +236,6 @@ public function drop(string $identifier, ?string $version = NULL, bool $local_re } } - /** - * Get a list of all stored importers and filefetchers, and their status. - * - * @return array - * The importer list object. - */ - public function list() { - return $this->importInfoList->buildList(); - } - /** * Summary. */ diff --git a/modules/datastore/src/Plugin/QueueWorker/ImportJob.php b/modules/datastore/src/Plugin/QueueWorker/ImportJob.php index be9ca6c6b9..1a43e227c1 100644 --- a/modules/datastore/src/Plugin/QueueWorker/ImportJob.php +++ b/modules/datastore/src/Plugin/QueueWorker/ImportJob.php @@ -219,10 +219,10 @@ protected function runIt() { $this->store(); if ($this->getBytesProcessed() >= $size) { - $this->getResult()->setStatus(Result::DONE); + $this->setStatus(Result::DONE); } else { - $this->getResult()->setStatus(Result::STOPPED); + $this->setStatus(Result::STOPPED); } return $this->getResult(); @@ -258,8 +258,8 @@ protected function assertTextFile(string $filename) { * Updated result object. */ protected function setResultError($message): Result { - $this->getResult()->setStatus(Result::ERROR); $this->getResult()->setError($message); + $this->setStatus(Result::ERROR); return $this->getResult(); } @@ -291,7 +291,7 @@ protected function parseAndStore($filename, $maximumExecutionTime) { $chunk = fread($h, self::BYTES_PER_CHUNK); if (!$chunk) { - $this->getResult()->setStatus(Result::DONE); + $this->setStatus(Result::DONE); $this->parser->finish(); break; } @@ -299,8 +299,8 @@ protected function parseAndStore($filename, $maximumExecutionTime) { $this->parser->feed($chunk); $chunksProcessed++; - $this->store(); $this->setStateProperty('chunksProcessed', $chunksProcessed); + $this->store(); } fclose($h); } diff --git a/modules/datastore/src/Service/Factory/ImportServiceFactory.php b/modules/datastore/src/Service/Factory/ImportServiceFactory.php index 905fdd8604..0a54f9ff91 100644 --- a/modules/datastore/src/Service/Factory/ImportServiceFactory.php +++ b/modules/datastore/src/Service/Factory/ImportServiceFactory.php @@ -52,12 +52,7 @@ public function getInstance(string $identifier, array $config = []) { } $resource = $config['resource']; - - if (!isset($this->services[$identifier])) { - $this->services[$identifier] = new ImportService($resource, $this->jobStoreFactory, $this->databaseTableFactory); - } - - return $this->services[$identifier]; + return new ImportService($resource, $this->jobStoreFactory, $this->databaseTableFactory); } } diff --git a/modules/datastore/src/Service/ImportService.php b/modules/datastore/src/Service/ImportService.php index 1a62cf9385..bf4cbf9ec2 100644 --- a/modules/datastore/src/Service/ImportService.php +++ b/modules/datastore/src/Service/ImportService.php @@ -13,7 +13,11 @@ use Procrastinator\Result; /** - * Datastore import service. + * Datastore importer. + * + * @todo This class has state and is not actually a service because it holds + * state. Have import() take an argument of a resource, instead of storing it + * as a property. */ class ImportService { use LoggerTrait; @@ -43,9 +47,9 @@ class ImportService { /** * The DKAN Resource to import. * - * @var \Drupal\common\DataResource + * @var \Drupal\common\DataResource|null */ - private $resource; + private ?DataResource $resource; /** * The jobstore factory service. @@ -54,14 +58,25 @@ class ImportService { * * @todo Can we remove this? */ - private $jobStoreFactory; + private JobStoreFactory $jobStoreFactory; /** * Database table factory service. * * @var \Drupal\datastore\Storage\DatabaseTableFactory */ - private $databaseTableFactory; + private DatabaseTableFactory $databaseTableFactory; + + /** + * Import job for the current import. + * + * Access using self::getImporter(). + * + * @var \Drupal\datastore\Plugin\QueueWorker\ImportJob|null + * + * @see self::getImporter() + */ + private ?ImportJob $importJob; /** * Create a resource service instance. @@ -100,13 +115,10 @@ protected function getResource(): DataResource { * Import. */ public function import() { - $importer = $this->getImporter(); - $importer->run(); + $result = $this->getImporter()->run(); - $result = $this->getResult(); - $resource = $this->getResource(); if ($result->getStatus() === Result::ERROR) { - $datastore_resource = $resource->getDatastoreResource(); + $datastore_resource = $this->getResource()->getDatastoreResource(); $this->setLoggerFactory(\Drupal::service('logger.factory')); $this->error('Error importing resource id:%id path:%path message:%message', [ '%id' => $datastore_resource->getId(), @@ -115,31 +127,28 @@ public function import() { ]); } // If the import job finished successfully... + // @todo This should be an event that is emitted, and then processed + // elsewhere. elseif ($result->getStatus() === Result::DONE) { // Queue the imported resource for post-import processing. $post_import_queue = \Drupal::service('queue')->get('post_import'); - $post_import_queue->createItem($resource); + $post_import_queue->createItem($this->getResource()); } } - /** - * Get result. - */ - public function getResult(): Result { - $importer = $this->getImporter(); - return $importer->getResult(); - } - /** * Build an Importer. * - * @return \Drupal\datastore\Import + * @return \Drupal\datastore\Plugin\QueueWorker\ImportJob * Importer. * * @throws \Exception - * Throws exception if cannot create valid importer object. + * Throws exception if we cannot create a valid importer object. */ public function getImporter(): ImportJob { + if ($this->importJob ?? FALSE) { + return $this->importJob; + } $datastore_resource = $this->getResource()->getDatastoreResource(); $delimiter = ","; @@ -147,7 +156,7 @@ public function getImporter(): ImportJob { $delimiter = "\t"; } - $importer = call_user_func([$this->importerClass, 'get'], + $this->importJob = call_user_func([$this->importerClass, 'get'], $datastore_resource->getId(), $this->jobStoreFactory->getInstance(ImportJob::class), [ @@ -157,9 +166,9 @@ public function getImporter(): ImportJob { ] ); - $importer->setTimeLimit(self::DEFAULT_TIMELIMIT); + $this->importJob->setTimeLimit(self::DEFAULT_TIMELIMIT); - return $importer; + return $this->importJob; } /** diff --git a/modules/datastore/src/Service/Info/ImportInfo.php b/modules/datastore/src/Service/Info/ImportInfo.php index fde891bc1b..48b04989c8 100644 --- a/modules/datastore/src/Service/Info/ImportInfo.php +++ b/modules/datastore/src/Service/Info/ImportInfo.php @@ -2,25 +2,21 @@ namespace Drupal\datastore\Service\Info; +use Drupal\common\DataResource; +use Drupal\datastore\DatastoreService; use Drupal\datastore\Plugin\QueueWorker\ImportJob; -use Drupal\common\Storage\JobStoreFactory; use Drupal\datastore\Service\Factory\ImportFactoryInterface; use Drupal\datastore\Service\ResourceLocalizer; +use Drupal\metastore\ResourceMapper; use FileFetcher\FileFetcher; use Procrastinator\Job\Job; +use Procrastinator\Result; /** * Defines and provide a single item for an ImportInfoList. */ class ImportInfo { - /** - * A JobStore object. - * - * @var \Drupal\common\Storage\JobStoreFactory - */ - private $jobStoreFactory; - /** * Resource localizer service. * @@ -36,19 +32,48 @@ class ImportInfo { private $importServiceFactory; /** - * FileFetcher service. + * Resource mapper service. + * + * @var \Drupal\metastore\ResourceMapper + */ + private ResourceMapper $resourceMapper; + + /** + * Default values for import status. * - * @var \FileFetcher\FileFetcher + * @var array */ - private $fileFetcher; + protected static $defaultItemValues = [ + 'fileName' => '', + 'fileFetcherStatus' => Result::WAITING, + 'fileFetcherBytes' => 0, + 'fileFetcherPercentDone' => 0, + 'importerStatus' => Result::WAITING, + 'importerBytes' => 0, + 'importerPercentDone' => 0, + 'importerError' => NULL, + ]; + + /** + * Datastore service. + * + * @var \Drupal\datastore\DatastoreService + */ + protected DatastoreService $datastoreService; /** * Constructor. */ - public function __construct(JobStoreFactory $jobStoreFactory, ResourceLocalizer $resourceLocalizer, ImportFactoryInterface $importServiceFactory) { - $this->jobStoreFactory = $jobStoreFactory; + public function __construct( + ResourceLocalizer $resourceLocalizer, + ImportFactoryInterface $importServiceFactory, + ResourceMapper $resourceMapper, + DatastoreService $datastoreService + ) { $this->resourceLocalizer = $resourceLocalizer; $this->importServiceFactory = $importServiceFactory; + $this->resourceMapper = $resourceMapper; + $this->datastoreService = $datastoreService; } /** @@ -60,68 +85,56 @@ public function __construct(JobStoreFactory $jobStoreFactory, ResourceLocalizer * Resource version. * * @return object - * And object with info about imports: file name, fetching status, etc. + * An object with info about imports: file name, fetching status, etc. */ public function getItem(string $identifier, string $version) { - [$ff, $imp] = $this->getFileFetcherAndImporter($identifier, $version); - - $item = (object) [ - 'fileName' => '', - 'fileFetcherStatus' => 'waiting', - 'fileFetcherBytes' => 0, - 'fileFetcherPercentDone' => 0, - 'importerStatus' => 'waiting', - 'importerBytes' => 0, - 'importerPercentDone' => 0, - 'importerError' => NULL, - ]; - - if (isset($ff)) { - $this->fileFetcher = $ff; - $item->fileName = $this->getFileName($ff); - $item->fileFetcherStatus = $ff->getResult()->getStatus(); - $item->fileFetcherBytes = $this->getBytesProcessed($ff); - $item->fileFetcherPercentDone = $this->getPercentDone($ff); - } + $item = (object) static::$defaultItemValues; + + if ($resource = $this->resourceMapper->get($identifier, ResourceLocalizer::LOCAL_FILE_PERSPECTIVE, $version)) { + /** @var \FileFetcher\FileFetcher $ff */ + if ($ff = $this->getFileFetcher($resource)) { + $item->fileName = $this->getFileName($ff); + $item->fileFetcherStatus = $ff->getResult()->getStatus(); + $item->fileFetcherBytes = $this->getBytesProcessed($ff); + $item->fileFetcherPercentDone = $this->getPercentDone($ff); + } - /** @var \Drupal\datastore\Plugin\QueueWorker\ImportJob $imp */ - if (isset($imp)) { - $item->importerStatus = $imp->getResult()->getStatus(); - $item->importerError = $imp->getResult()->getError(); - $item->importerBytes = $this->getBytesProcessed($imp); - $item->importerPercentDone = $this->getPercentDone($imp); + /** @var \Drupal\datastore\Plugin\QueueWorker\ImportJob $import_job */ + if ($import_job = $this->getImporter($resource)) { + $item->importerStatus = $import_job->getResult()->getStatus(); + $item->importerError = $import_job->getResult()->getError(); + $item->importerBytes = $this->getBytesProcessed($import_job); + $item->importerPercentDone = $this->getPercentDone($import_job); + } } - return (object) $item; + return $item; } /** - * Get the filefetcher and importer objects for a resource. + * Get a file fetcher for the given resource. * - * @param string $identifier - * Resource identifier. - * @param string $version - * Resource version. + * @param \Drupal\common\DataResource $resource + * Resource to get the file fetcher for. * - * @return array - * Array with a filefetcher and importer object. + * @return \FileFetcher\FileFetcher + * File fetcher object for the resource. */ - protected function getFileFetcherAndImporter($identifier, $version) { - try { - $resource = $this->resourceLocalizer->get($identifier, $version); - - if ($resource) { - $fileFetcher = $this->resourceLocalizer->getFileFetcher($resource); - - $importer = $this->importServiceFactory->getInstance($resource->getUniqueIdentifier(), - ['resource' => $resource])->getImporter(); + protected function getFileFetcher(DataResource $resource): FileFetcher { + return $this->resourceLocalizer->getFileFetcher($resource); + } - return [$fileFetcher, $importer]; - } - } - catch (\Exception $e) { - } - return [NULL, NULL]; + /** + * Get an import job store object for the resource. + * + * @param \Drupal\common\DataResource $resource + * Resource object reperesenting the resource. + * + * @return \Drupal\datastore\Plugin\QueueWorker\ImportJob + * Import + */ + protected function getImporter(DataResource $resource): ImportJob { + return $this->datastoreService->getImportService($resource)->getImporter(); } /** @@ -144,7 +157,7 @@ private function getFileName($fileFetcher): string { */ private function getPercentDone(Job $job): float { $bytes = $this->getBytesProcessed($job); - $filesize = $this->getFileSize(); + $filesize = $this->getFileSize($job); return ($filesize > 0) ? round($bytes / $filesize * 100) : 0; } @@ -154,34 +167,32 @@ private function getPercentDone(Job $job): float { * @return int * File size in bytes. */ - protected function getFileSize(): int { - return $this->fileFetcher->getStateProperty('total_bytes'); + protected function getFileSize(Job $job): int { + return $job->getStateProperty('total_bytes'); } /** * Calculate bytes processed based on chunks processed in the importer data. * * @param \Procrastinator\Job\Job $job - * Either a FileFetcher or Importer object. + * Job object. In practice this will be either an ImportJob, a FileFetcher, + * or one of their subclasses. * * @return int * Total bytes processed. */ protected function getBytesProcessed(Job $job): int { - $className = get_class($job); - switch ($className) { - // For Importer, avoid going above total size due to chunk multiplication. - case ImportJob::class: - $chunksSize = $job->getStateProperty('chunksProcessed') * ImportJob::BYTES_PER_CHUNK; - $fileSize = $this->getFileSize(); - return ($chunksSize > $fileSize) ? $fileSize : $chunksSize; - - case FileFetcher::class: - return $job->getStateProperty('total_bytes_copied'); - - default: - return 0; + // Handle ImportJob and its subclasses. + if (is_a($job, ImportJob::class)) { + $chunksSize = $job->getStateProperty('chunksProcessed') * ImportJob::BYTES_PER_CHUNK; + $fileSize = $this->getFileSize($job); + return ($chunksSize > $fileSize) ? $fileSize : $chunksSize; + } + // Handle FileFetcher and its subclasses. + if (is_a($job, FileFetcher::class)) { + return $job->getStateProperty('total_bytes_copied'); } + return 0; } } diff --git a/modules/datastore/src/Service/Info/ImportInfoList.php b/modules/datastore/src/Service/Info/ImportInfoList.php index 9ddec0c7ce..a74aea6fb9 100644 --- a/modules/datastore/src/Service/Info/ImportInfoList.php +++ b/modules/datastore/src/Service/Info/ImportInfoList.php @@ -52,8 +52,8 @@ public function __construct(JobStoreFactory $jobStoreFactory, ImportInfo $import * @return array * An array of ImportInfo objects, keyed by UUID. * - * @todo Going directly to get filefetcher objects does not feel right. - * We should have cleaner interfaces to get the data we need. + * @todo This method assumes a 1:1 relationship between filefetcher job stores + * and status to report. This might not be the case. */ public function buildList() { $list = []; @@ -61,7 +61,7 @@ public function buildList() { $store = $this->jobStoreFactory->getInstance(FileFetcher::class); foreach ($store->retrieveAll() as $id) { - $pieces = explode("_", $id); + $pieces = explode('_', $id); // The filefetcher identifier for resources has the form _ // by doing this check we can eliminate processing some unrelated file diff --git a/modules/datastore/tests/src/Kernel/Controller/ImportControllerTest.php b/modules/datastore/tests/src/Kernel/Controller/ImportControllerTest.php new file mode 100644 index 0000000000..514adf4590 --- /dev/null +++ b/modules/datastore/tests/src/Kernel/Controller/ImportControllerTest.php @@ -0,0 +1,39 @@ +container); + $request = Request::create('http://blah/api'); + $result = $webServiceApi->import($request); + + $this->assertInstanceOf(JsonResponse::class, $result); + } + +} diff --git a/modules/datastore/tests/src/Kernel/Plugin/QueueWorker/ImportQueueWorkerTest.php b/modules/datastore/tests/src/Kernel/Plugin/QueueWorker/ImportQueueWorkerTest.php index 90cde4d2d4..a5917e8247 100644 --- a/modules/datastore/tests/src/Kernel/Plugin/QueueWorker/ImportQueueWorkerTest.php +++ b/modules/datastore/tests/src/Kernel/Plugin/QueueWorker/ImportQueueWorkerTest.php @@ -42,7 +42,6 @@ public function testErrorPath() { $this->container->get('dkan.datastore.service.factory.import'), $this->container->get('queue'), $this->container->get('dkan.common.job_store'), - $this->container->get('dkan.datastore.import_info_list'), $this->container->get('dkan.datastore.service.resource_processor.dictionary_enforcer'), ]) ->onlyMethods(['import']) @@ -91,7 +90,6 @@ public function testRequeue() { $this->container->get('dkan.datastore.service.factory.import'), $this->container->get('queue'), $this->container->get('dkan.common.job_store'), - $this->container->get('dkan.datastore.import_info_list'), $this->container->get('dkan.datastore.service.resource_processor.dictionary_enforcer'), ]) ->onlyMethods(['import']) @@ -230,7 +228,6 @@ public function testAlreadyImported() { $this->container->get('dkan.datastore.service.factory.import'), $this->container->get('queue'), $this->container->get('dkan.common.job_store'), - $this->container->get('dkan.datastore.import_info_list'), $this->container->get('dkan.datastore.service.resource_processor.dictionary_enforcer'), ]) ->onlyMethods(['getStorage']) diff --git a/modules/datastore/tests/src/Kernel/Service/ImportServiceTest.php b/modules/datastore/tests/src/Kernel/Service/ImportServiceTest.php new file mode 100644 index 0000000000..9d0311303c --- /dev/null +++ b/modules/datastore/tests/src/Kernel/Service/ImportServiceTest.php @@ -0,0 +1,106 @@ +setStatus(Result::DONE); + + $import_job = $this->getMockBuilder(ImportJob::class) + ->onlyMethods(['run', 'getResult']) + ->disableOriginalConstructor() + ->getMock(); + $import_job->method('run') + ->willReturn($result); + $import_job->method('getResult') + ->willReturn($result); + + /** @var \Drupal\datastore\Service\ImportService $import_service */ + $import_service = $this->getMockBuilder(ImportService::class) + ->onlyMethods(['getImporter']) + ->setConstructorArgs([ + new DataResource('abc.txt', 'text/csv'), + $this->container->get('dkan.common.job_store'), + $this->container->get('dkan.datastore.database_table_factory'), + ]) + ->getMock(); + $import_service->method('getImporter') + ->willReturn($import_job); + + // There should be zero queue items to start with. + /** @var \Drupal\Core\Queue\QueueInterface $queue */ + $queue = $this->container->get('queue')->get('post_import'); + $this->assertEquals(0, $queue->numberOfItems()); + + // Perform the import. + $import_service->import(); + + $this->assertEquals( + Result::DONE, + $import_service->getImporter()->getResult()->getStatus() + ); + + // Did we add a queue item? + $this->assertEquals(1, $queue->numberOfItems()); + } + + /** + * @covers ::import + */ + public function testLogImportError() { + // Tell the logger channel factory to use a buffering logger. + $logger = new BufferingLogger(); + $logger_factory = $this->createMock(LoggerChannelFactory::class); + $logger_factory->expects($this->once()) + ->method('get') + ->with('dkan') + ->willReturn($logger); + $this->container->set('logger.factory', $logger_factory); + + // Get an import service. + /** @var \Drupal\datastore\Service\Factory\ImportServiceFactory $import_service_factory */ + $import_service_factory = $this->container->get('dkan.datastore.service.factory.import'); + $import_service = $import_service_factory->getInstance('id', [ + 'resource' => new DataResource('abc.txt', 'text/csv'), + ]); + + $import_service->import(); + + // Ensure the log entry was created. + $this->assertEquals( + 'Error importing resource id:%id path:%path message:%message', + $logger->cleanLogs()[0][1] + ); + } + +} diff --git a/modules/datastore/tests/src/Kernel/Service/Info/ImportInfoListTest.php b/modules/datastore/tests/src/Kernel/Service/Info/ImportInfoListTest.php new file mode 100644 index 0000000000..4f1b58cdd1 --- /dev/null +++ b/modules/datastore/tests/src/Kernel/Service/Info/ImportInfoListTest.php @@ -0,0 +1,82 @@ +add(ResourceMapper::class, 'get', DataResource::class) + ->getMock(); + $this->container->set('dkan.metastore.resource_mapper', $resource_mapper); + + $import_job = (new Chain($this)) + ->add(ImportJob::class, 'getResult', $result) + ->getMock(); + + $import_info = $this->getMockBuilder(ImportInfo::class) + ->onlyMethods([ + 'getImporter', + 'getFileFetcher', + 'getBytesProcessed', + ]) + ->setConstructorArgs([ + $this->container->get('dkan.datastore.service.resource_localizer'), + $this->container->get('dkan.datastore.service.factory.import'), + $this->container->get('dkan.metastore.resource_mapper'), + $this->container->get('dkan.datastore.service'), + ]) + ->getMock(); + $import_info->method('getImporter')->willReturn($import_job); + $import_info->method('getFileFetcher')->willReturn($ff); + $import_info->method('getBytesProcessed')->willReturn(1500); + + $this->container->set('dkan.datastore.import_info', $import_info); + + $job_store = (new Chain($this)) + ->add(JobStore::class, 'retrieveAll', ['1_1']) + ->getMock(); + + $job_store_factory = (new Chain($this)) + ->add(JobStoreFactory::class, 'getInstance', $job_store) + ->getMock(); + + $this->container->set('dkan.common.job_store', $job_store_factory); + + // Build the list. + /** @var \Drupal\datastore\Service\Info\ImportInfoList $import_info_list */ + $import_info_list = $this->container->get('dkan.datastore.import_info_list'); + $list = $import_info_list->buildList(); + + // Assert the output. + $this->assertStringContainsString('File import error', $list['1_1']->importerError); + $this->assertStringContainsString('error', $list['1_1']->importerStatus); + $this->assertEquals(1500, $list['1_1']->importerBytes); + } + +} diff --git a/modules/datastore/tests/src/Kernel/Service/Info/ImportInfoTest.php b/modules/datastore/tests/src/Kernel/Service/Info/ImportInfoTest.php new file mode 100644 index 0000000000..1cc1b846e5 --- /dev/null +++ b/modules/datastore/tests/src/Kernel/Service/Info/ImportInfoTest.php @@ -0,0 +1,130 @@ +container->get('dkan.metastore.resource_mapper'); + $mapper->register($source_resource); + $this->assertInstanceOf( + DataResource::class, + $source_resource = $mapper->get($source_resource->getIdentifier()) + ); + // Our localized perspective does not yet exist. + $this->assertNull( + $mapper->get($source_resource->getIdentifier(), ResourceLocalizer::LOCAL_FILE_PERSPECTIVE) + ); + + /** @var \Drupal\datastore\Service\Info\ImportInfo $import_info */ + $import_info = $this->container->get('dkan.datastore.import_info'); + + // Gather the item info before localization. + $import_info_item = $import_info->getItem( + $source_resource->getIdentifier(), + $source_resource->getVersion() + ); + // Not done, and bytes are zero. + $this->assertEquals('waiting', $import_info_item->fileFetcherStatus); + // Fair to say there are no bytes yet since we haven't localized. + $this->assertEquals(0, $import_info_item->fileFetcherBytes); + + // OK, let's localize it. + /** @var \Drupal\datastore\Service\ResourceLocalizer $resource_localizer */ + $resource_localizer = $this->container->get('dkan.datastore.service.resource_localizer'); + // Try to localize. + $this->assertInstanceOf( + Result::class, + $result = $resource_localizer->localize($source_resource->getIdentifier()) + ); + $this->assertEquals(Result::DONE, $result->getStatus(), $result->getData()); + + // What about our local perspective? + $this->assertInstanceOf( + DataResource::class, + $local_resource = $resource_localizer->get( + $source_resource->getIdentifier(), + $source_resource->getVersion(), + ResourceLocalizer::LOCAL_FILE_PERSPECTIVE + ) + ); + // Does the localized file actually exist? + $this->assertFileExists($local_resource->getFilePath()); + $this->assertNotEmpty(file_get_contents($local_resource->getFilePath())); + + // Get a file fetcher job from the resource localizer. Does it report the + // correct file size? + $ff = $resource_localizer->getFileFetcher($source_resource); + $ff_state = $ff->getState(); + $this->assertEquals( + filesize($local_resource->getFilePath()), + $ff_state['total_bytes'] + ); + $this->assertEquals( + $ff_state['total_bytes'], + $ff_state['total_bytes_copied'] + ); + + // Let's now examine the import job + /** @var \Drupal\datastore\DatastoreService $datastore_service */ + $datastore_service = $this->container->get('dkan.datastore.service'); + /** @var \Drupal\datastore\Service\ImportService $import_service */ + $import_service = $datastore_service->getImportService($local_resource); + /** @var \Drupal\datastore\Plugin\QueueWorker\ImportJob $import_job */ + $import_job = $import_service->getImporter(); + $import_result = $import_job->getResult()->getStatus(); + $this->assertEquals(Result::WAITING, $import_result); + + // ImportInfo::getItem() ultimately calls + // ResourceLocalizer::getFileFetcher() just like we did above. Do its + // results match? + $import_info_item = $import_info->getItem( + $source_resource->getIdentifier(), + $source_resource->getVersion() + ); + // Is it done? + $this->assertEquals(Result::DONE, $import_info_item->fileFetcherStatus); + // Do we report the correct number of bytes? + $this->assertEquals( + filesize($local_resource->getFilePath()), + $import_info_item->fileFetcherBytes + ); + } + +} diff --git a/modules/datastore/tests/src/Unit/Controller/ImportControllerTest.php b/modules/datastore/tests/src/Unit/Controller/ImportControllerTest.php deleted file mode 100644 index 8250d617f9..0000000000 --- a/modules/datastore/tests/src/Unit/Controller/ImportControllerTest.php +++ /dev/null @@ -1,63 +0,0 @@ -getContainer(); - - $webServiceApi = ImportController::create($container); - $request = Request::create("http://blah/api"); - $result = $webServiceApi->import($request); - - $this->assertTrue($result instanceof JsonResponse); - } - - /** - * Private. - */ - private function getContainer() { - $options = (new Options()) - ->add("dkan.datastore.service", DatastoreService::class) - ->add('dkan.metastore.metastore_item_factory', NodeDataFactory::class) - ->add('dkan.metastore.api_response', MetastoreApiResponse::class) - ->add('dkan.metastore.reference_lookup', ReferenceLookup::class) - ->index(0); - - return (new Chain($this)) - ->add(Container::class, "get", $options) - ->add(DatastoreService::class, "drop", NULL) - ->add(DatastoreService::class, "import", []) - ->add(RequestStack::class, 'getCurrentRequest', Request::class) - ->add(Request::class, 'getContent', json_encode((object) ['resource_ids' => ["1", "2"]])) - ->add(MetastoreApiResponse::class, 'getMetastoreItemFactory', NodeDataFactory::class) - ->add(MetastoreApiResponse::class, 'addReferenceDependencies', NULL) - ->add(NodeDataFactory::class, 'getInstance', Data::class) - ->add(Data::class, 'getCacheContexts', ['url']) - ->add(Data::class, 'getCacheTags', ['node:1']) - ->add(Data::class, 'getCacheMaxAge', 0) - ->getMock(); - } - -} diff --git a/modules/datastore/tests/src/Unit/DatastoreServiceTest.php b/modules/datastore/tests/src/Unit/DatastoreServiceTest.php index 91b9b48003..9b4bba21d2 100644 --- a/modules/datastore/tests/src/Unit/DatastoreServiceTest.php +++ b/modules/datastore/tests/src/Unit/DatastoreServiceTest.php @@ -4,6 +4,7 @@ use Drupal\Component\EventDispatcher\ContainerAwareEventDispatcher; use Drupal\Core\Queue\QueueFactory; +use Drupal\datastore\Plugin\QueueWorker\ImportJob; use Drupal\Tests\common\Traits\ServiceCheckTrait; use Drupal\common\DataResource; use Drupal\common\Storage\JobStore; @@ -42,7 +43,8 @@ public function testImport() { ->add(ResourceMapper::class, 'get', $resource) ->add(ImportServiceFactory::class, "getInstance", ImportService::class) ->add(ImportService::class, "import", NULL) - ->add(ImportService::class, "getResult", new Result()) + ->add(ImportService::class, 'getImporter', ImportJob::class) + ->add(ImportJob::class, 'getResult', new Result()) ->add(QueueFactory::class, "get", NULL) ->add(ContainerAwareEventDispatcher::class, "dispatch", NULL); diff --git a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php index bc078bdc32..3233738c52 100644 --- a/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php +++ b/modules/datastore/tests/src/Unit/Plugin/QueueWorker/ImportJobTest.php @@ -22,6 +22,11 @@ */ class ImportJobTest extends TestCase { + /** + * Database. + * + * @var \Drupal\common\Storage\DatabaseTableInterface + */ private $database; /** @@ -60,7 +65,7 @@ public function testBasics() { $datastore = $this->getDatastore($resource); $this->assertTrue($datastore->getParser() instanceof ParserInterface); - $this->assertEquals(Result::STOPPED, $datastore->getResult()->getStatus()); + $this->assertEquals(Result::WAITING, $datastore->getResult()->getStatus()); $datastore->run(); $this->assertNotEquals(Result::ERROR, $datastore->getResult()->getStatus()); diff --git a/modules/datastore/tests/src/Unit/Service/ImportServiceTest.php b/modules/datastore/tests/src/Unit/Service/ImportServiceTest.php deleted file mode 100644 index 95eaa6d8bd..0000000000 --- a/modules/datastore/tests/src/Unit/Service/ImportServiceTest.php +++ /dev/null @@ -1,162 +0,0 @@ -add('event_dispatcher', ContainerAwareEventDispatcher::class) - ->add('request_stack', RequestStack::class) - ->add('stream_wrapper_manager', StreamWrapperManager::class) - ->add('queue', QueueFactory::class) - ->index(0); - $container_chain = (new Chain($this)) - ->add(Container::class, 'get', $options) - ->add(StreamWrapperManager::class, 'getViaUri', PublicStream::class) - ->add(PublicStream::class, 'getExternalUrl', self::HOST) - ->add(QueueFactory::class, 'get', QueueInterface::class) - ->add(QueueInterface::class, 'createItem', NULL, 'items'); - - \Drupal::setContainer($container_chain->getMock()); - - $resource = new DataResource("http://hello.goodby/text.csv", "text/csv"); - - $result = (new Chain($this)) - ->add(Result::class, 'getStatus', Result::DONE) - ->getMock(); - - $jobStore = (new Chain($this)) - ->add(JobStore::class, "retrieve", "") - ->add(ImportJob::class, "run", Result::class) - ->add(ImportJob::class, "getResult", $result) - ->add(JobStore::class, "store", "") - ->getMock(); - - $databaseTableFactory = (new Chain($this)) - ->add(DatabaseTableFactory::class, "getInstance", DatabaseTable::class) - ->getMock(); - - $jobStoreFactory = (new Chain($this)) - ->add(JobStoreFactory::class, "getInstance", $jobStore) - ->getMock(); - - $service = new ImportService($resource, $jobStoreFactory, $databaseTableFactory); - $service->import(); - - $result = $service->getResult(); - $this->assertTrue($result instanceof Result); - $this->assertEmpty($result->getError()); - } - - /** - * Test a dictionary enforcer queue job is created on success when enabled. - */ - public function testDictEnforcerQueuedOnSuccess() { - $datastore_table = 'datastore_test'; - $resource = new DataResource('abc.txt', 'text/csv'); - - $options = (new Options()) - ->add('dkan.metastore.data_dictionary_discovery', DataDictionaryDiscoveryInterface::class) - ->add('logger', LoggerChannelFactory::class) - ->add('queue', QueueFactory::class) - ->index(0); - $containerChain = (new Chain($this)) - ->add(Container::class, 'get', $options) - ->add(LoggerChannelFactory::class, 'get', LoggerChannel::class) - ->add(LoggerChannel::class, 'error', NULL, 'errors') - ->add(QueueFactory::class, 'get', QueueInterface::class) - ->add(QueueInterface::class, 'createItem', NULL, 'items') - ->add(DataDictionaryDiscoveryInterface::class, 'getDataDictionaryMode', DataDictionaryDiscoveryInterface::MODE_SITEWIDE); - $container = $containerChain->getMock(); - \Drupal::setContainer($container); - - $importService = (new Chain($this)) - ->add(ImportService::class, 'getResource', DataResource::class) - ->add(ImportService::class, 'getImporter', ImportJob::class) - ->add(ImportService::class, 'getStorage', DatabaseTable::class) - ->add(ImportService::class, 'getResource', $resource) - ->add(ImportJob::class, 'run', Result::class) - ->add(ImportService::class, 'getResult', Result::class) - ->add(Result::class, 'getStatus', Result::DONE) - ->add(DatabaseTable::class, 'getTableName', $datastore_table) - ->getMock(); - $importService->import(); - - // Validate that a dictionary enforcer queue item was created. - $this->assertEquals([$resource], $containerChain->getStoredInput('items')); - } - - /** - * - */ - public function testLogImportError() { - $importMock = (new Chain($this)) - ->add(ImportService::class, 'getResource', new DataResource('abc.txt', 'text/csv')) - ->add(ImportService::class, 'getImporter', ImportJob::class) - ->add(ImportJob::class, 'run', Result::class) - ->add(ImportService::class, 'getResult', Result::class) - ->add(Result::class, 'getStatus', Result::ERROR) - ->getMock(); - - // Construct and set `\Drupal::container` mock. - $options = (new Options()) - ->add('stream_wrapper_manager', StreamWrapperManager::class) - ->add('logger.factory', LoggerChannelFactory::class) - ->index(0); - - $containerChain = (new Chain($this)) - ->add(Container::class, 'get', $options) - ->add(LoggerChannelFactory::class, 'get', LoggerChannel::class) - ->add(LoggerChannel::class, 'error', NULL, 'errors') - ->add(StreamWrapperManager::class, 'getViaUri', PublicStream::class) - ->add(PublicStream::class, 'getExternalUrl', self::HOST); - $container = $containerChain->getMock(); - - \Drupal::setContainer($container); - - $importMock->import(); - - $expectedLogError = 'Error importing resource id:%id path:%path message:%message'; - - $this->assertEquals($expectedLogError, $containerChain->getStoredInput('errors')[0]); - } - -} diff --git a/modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php b/modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php deleted file mode 100644 index 7f4805ef0a..0000000000 --- a/modules/datastore/tests/src/Unit/Service/Info/ImportInfoListTest.php +++ /dev/null @@ -1,48 +0,0 @@ -add(ImportJob::class, "getResult", $result) - ->getMock(); - - $services = (new Options()) - ->add('dkan.common.job_store', JobStoreFactory::class) - ->add('dkan.datastore.import_info', ImportInfo::class) - ->index(0); - - $container = (new Chain($this)) - ->add(Container::class, 'get', $services) - ->add(JobStoreFactory::class, 'getInstance', JobStore::class) - ->add(JobStore::class, 'retrieveAll', ["1_1"]) - ->add(ImportInfo::class, 'getFileFetcherAndImporter', [$ff, $imp]) - ->add(ImportInfo::class, 'getBytesProcessed', 1500) - ->getMock(); - - $listService = ImportInfoList::create($container); - $list = $listService->buildList(); - $this->assertStringContainsString('File import error', $list['1_1']->importerError); - $this->assertStringContainsString('error', $list['1_1']->importerStatus); - $this->assertEquals(1500, $list['1_1']->importerBytes); - } - -} diff --git a/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php b/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php index 14a55137bb..4cfd7e057b 100644 --- a/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php +++ b/modules/datastore/tests/src/Unit/Service/ResourceLocalizerTest.php @@ -35,7 +35,6 @@ class ResourceLocalizerTest extends TestCase { * Test removal of a local resource file. */ public function testResourceLocalizerRemove(): void { - $this->markTestIncomplete('Convert this to a kernel test.'); $this->callWithTmpFile([$this, 'doTestResourceLocalizerRemove']); }