diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index a05e4dc..9a7419a 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -19,6 +19,7 @@ use Psr\Http\Message\ServerRequestInterface; use Psr\Log\LoggerInterface; use Slub\LisztCommon\Common\ElasticClientBuilder; +use Slub\LisztBibliography\Exception\TooManyRequestsException; use Slub\LisztBibliography\Processing\BibEntryProcessor; use Slub\LisztBibliography\Processing\BibElasticMapping; use Symfony\Component\Console\Command\Command; @@ -41,6 +42,8 @@ class IndexCommand extends Command const API_TRIALS = 3; protected string $apiKey; + protected string $groupId; + protected Collection $bibliographyItems; protected Collection $deletedItems; protected Collection $teiDataSets; @@ -103,6 +106,16 @@ protected function initialize(InputInterface $input, OutputInterface $output): v $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography'); $this->client = ElasticClientBuilder::getClient(); $this->apiKey = $this->extConf['zoteroApiKey']; + if ($this->apiKey == '') { + $this->logger->info('Please set an API key in the extension configuration.'); + throw new \Exception('Please set an API key in the extension configuration.'); + } + $this->groupId = $this->extConf['zoteroGroupId']; + if ($this->groupId == '') { + $this->logger->info('Please set a group ID in the extension configuration.'); + throw new \Exception('Please set a group ID in the extension configuration.'); + } + $this->io = new SymfonyStyle($input, $output); $this->io->title($this->getDescription()); } @@ -125,9 +138,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int protected function fullSync(InputInterface $input): void { - $client = new ZoteroApi($this->extConf['zoteroApiKey']); + $client = GeneralUtility::makeInstance(ZoteroApi::class, $this->extConf['zoteroApiKey']); $response = $client-> - group($this->extConf['zoteroGroupId'])-> + group($this->groupId)-> items()-> top()-> limit(1)-> @@ -160,7 +173,7 @@ protected function fullSync(InputInterface $input): void } else { $this->io->error("Exception: " . $e->getMessage()); $this->logger->error('Bibliography sync unsuccessful. Error creating elasticsearch index.'); - throw new \Exception('Bibliography sync unsuccessful.'); + throw new \Exception('Bibliography sync unsuccessful. Error creating elasticsearch index.'); } } @@ -175,6 +188,10 @@ protected function fullSync(InputInterface $input): void $advanceBy = min($remainingItems, $this->bulkSize); $this->io->progressAdvance($advanceBy); $cursor += $this->bulkSize; + } catch (TooManyRequestsException $e) { + $this->io->note('Received a 429 status from Zotero API. Too many requests. Please try again later.'); + $this->logger->error('Received a 429 status from Zotero API. Too many requests. Please try again later.'); + throw new TooManyRequestsException('Bibliography sync unsuccessful.'); } catch (\Exception $e) { $this->io->newline(1); $this->io->caution($e->getMessage()); @@ -182,7 +199,7 @@ protected function fullSync(InputInterface $input): void if ($apiCounter == 0) { $this->io->note('Giving up after ' . self::API_TRIALS . ' trials.'); $this->logger->error('Bibliography sync unsuccessful. Zotero API sent {trials} 500 errors.', ['trials' => self::API_TRIALS]); - throw new \Exception('Bibliography sync unsuccessful.'); + throw new \Exception('Bibliography sync unsuccessful. Zotero API sent {trials} 500 errors.', ['trials' => self::API_TRIALS]); } else { $this->io->note('Trying again. ' . --$apiCounter . ' trials left.'); } @@ -206,7 +223,7 @@ protected function versionedSync(int $version): void if ($apiCounter == 0) { $this->io->note('Giving up after ' . self::API_TRIALS . ' trials.'); $this->logger->warning('Bibliography sync unsuccessful. Zotero API sent {trials} 500 errors.', ['trials' => self::API_TRIALS]); - throw new \Exception('Bibliography sync unsuccessful.'); + throw new \Exception('Bibliography sync unsuccessful. Zotero API sent {trials} 500 errors.', ['trials' => self::API_TRIALS]); } else { $this->io->note('Trying again. ' . --$apiCounter . ' trials left.'); } @@ -268,16 +285,16 @@ protected function getVersion(InputInterface $input): int return 0; } else { $this->io->error("Exception: " . $e->getMessage()); - throw new \Exception('Bibliography sync unsuccessful.'); + throw new \Exception('Bibliography sync unsuccessful.' . $e->getMessage()); } } } protected function fetchBibliography(int $cursor, int $version): void { - $client = new ZoteroApi($this->extConf['zoteroApiKey']); + $client = GeneralUtility::makeInstance(ZoteroApi::class, $this->extConf['zoteroApiKey']); $response = $client-> - group($this->extConf['zoteroGroupId'])-> + group($this->groupId)-> items()-> top()-> start($cursor)-> @@ -285,6 +302,13 @@ protected function fetchBibliography(int $cursor, int $version): void setSince($version)-> send(); + if (isset($response->getHeaders()['Backoff'])) { + $this->logger->warning('Received a Backoff header: '. $response->getHeaders()['Backoff']); + } + if ($response->getStatusCode() == 429) { + throw new TooManyRequestsException('Too many requests issued. Try again later.'); + } + $this->bibliographyItems = Collection::wrap($response->getBody())-> pluck('data'); } @@ -297,10 +321,10 @@ protected function fetchCitations(int $cursor, int $version): void protected function fetchCitationLocale(string $locale, int $cursor, int $version): void { - $client = new ZoteroApi($this->extConf['zoteroApiKey']); + $client = GeneralUtility::makeInstance(ZoteroApi::class, $this->extConf['zoteroApiKey']); $style = $this->extConf['zoteroStyle']; $response = $client-> - group($this->extConf['zoteroGroupId'])-> + group($this->groupId)-> items()-> top()-> start($cursor)-> @@ -320,9 +344,9 @@ protected function fetchCitationLocale(string $locale, int $cursor, int $version protected function fetchTeiData(int $cursor, int $version): void { - $client = new ZoteroApi($this->extConf['zoteroApiKey']); + $client = GeneralUtility::makeInstance(ZoteroApi::class, $this->extConf['zoteroApiKey']); $response = $client-> - group($this->extConf['zoteroGroupId'])-> + group($this->groupId)-> items()-> top()-> start($cursor)-> diff --git a/Classes/Exception/TooManyRequestsException.php b/Classes/Exception/TooManyRequestsException.php new file mode 100644 index 0000000..623ce32 --- /dev/null +++ b/Classes/Exception/TooManyRequestsException.php @@ -0,0 +1,5 @@ +exampleEntry = + <<title", + "creators": [ + { + "creatorType": "author", + "firstName": "$this->authorFirstName", + "lastName": "$this->authorLastName" + }, + { + "creatorType": "editor", + "firstName": "$this->editorFirstName", + "lastName": "$this->editorLastName" + }, + { + "creatorType": "translator", + "firstName": "$this->translatorFirstName", + "lastName": "$this->translatorLastName" + } + ], + "place": "$this->place", + "volume": "$this->volume", + "numberOfVolumes": "$this->numberOfVolumes", + "date": "$this->date" + } + JSON; + + $siteFinderMock = $this->createMock(SiteFinder::class); + $loggerMock = $this->createMock(LoggerInterface::class); + $this->subject = GeneralUtility::makeInstance(IndexCommand::class, $siteFinderMock, $loggerMock); + } + + public function tearDown(): void + { + parent::tearDown(); + } + + /** + * @test + */ + public function commandBreaksOn429(): void + { + $response = $this->createStub(Response::class); + $client = $this->createStub(ZoteroApi::class); + $response->method('getStatusCode')-> + willReturn(429); + $client->method('send')-> + willReturn($response); +/* + $response = $this->getAccessibleMock(Response::class, ['getStatusCode'], [], '', false); + $response->method('getStatusCode')-> + willReturn(429); + $selfReturningMethods = [ 'group', 'items', 'top', 'start', 'limit', 'setSince' ]; + $client = $this->getAccessibleMock(ZoteroApi::class, + ['send', ...$selfReturningMethods ], + [], + '', + false + ); + $client->method('send')-> + willReturn($response); + foreach ($selfReturningMethods as $method) { + $client-> + method($method)-> + willReturn($client); + } +*/ + +/* + $indices = $this->createStub(Indices::class); + $indices->method('exists')-> + willReturn(true); +*/ + $elasticClient = $this->createStub(ClientInterface::class); + //$elasticClient->method('search')->willReturn(true); +/* + $elasticClient->method('indices')-> + willReturn($indices); + $elasticClient->method('exists')-> + willReturn(true); +*/ + $elasticClientBuilder = $this->createStub(ElasticClientBuilder::class); + $elasticClientBuilder->method('getClient')-> + willReturn($elasticClient); + GeneralUtility::addInstance(ElasticClientBuilder::class, $elasticClientBuilder); + + $this->expectException(TooManyRequestsException::class); + GeneralUtility::addInstance(ZoteroApi::class, $client); + + $extConfMap = [ + [ + 'liszt_bibliography', + '', + [ + 'zoteroApiKey' => 'abc', + 'zoteroGroupId' => 'abc', + 'zoteroBulkSize' => 10, + 'elasticIndexName' => 'zotero' + ] + ], + [ + 'liszt_common', + '', + [ + 'elasticHostName' => 'https://elasticsearch', + 'elasticCaFilePath' => '', + 'elasticPwdFileName' => '', + 'elasticCredentialsFilePath' => '', + 'zoteroBulkSize' => 10 + ] + ] + ]; + + $extConf = $this->createStub(ExtensionConfiguration::class); + $extConf->method('get')->willReturnMap($extConfMap); + GeneralUtility::addInstance(ExtensionConfiguration::class, $extConf); + GeneralUtility::addInstance(ExtensionConfiguration::class, $extConf); + + $inputMock = $this->createStub(InputInterface::class); + $outputMock = $this->createStub(OutputInterface::class); + + $this->subject->run($inputMock, $outputMock); + + } + + /** + * @test + */ + public function commandBreaksAfterThreeTimes500(): void + { + $book = $this->subject->process($this->exampleBookArray, new Collection(), new Collection()); + $bookSection = $this->subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); + $article = $this->subject->process($this->exampleArticleArray, new Collection(), new Collection()); + + self::assertEquals(Str::of($this->title), $book['tx_lisztcommon_body']); + self::assertEquals(Str::of($this->title), $bookSection['tx_lisztcommon_body']); + self::assertEquals(Str::of($this->title), $article['tx_lisztcommon_body']); + } + + /** + * @test + */ + public function unsetGroupIdLeadsToException(): void + { + $book = $this->subject->process($this->exampleBookArray, new Collection(), new Collection()); + $expected = Str::of( + 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . ', ' . + 'übers. von ' . $this->translatorFirstName . ' ' . $this->translatorLastName . ', ' . + $this->numberOfVolumes . 'Bde., ' . + 'Bd. ' . $this->volume . ', ' . + $this->place . ' ' . + $this->date + ); + + self::assertEquals($expected, $book['tx_lisztcommon_footer']); + } + + /** + * @test + */ + public function unsetApiKeyLeadsToException(): void + { + $bookSection = $this->subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); + $expected = Str::of( + 'In ' . $this->bookTitle . ', ' . + 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . ', ' . + 'übers. von ' . $this->translatorFirstName . ' ' . $this->translatorLastName . ', ' . + $this->numberOfVolumes . 'Bde., ' . + 'Bd. ' . $this->volume . ', ' . + $this->place . ' ' . + $this->date . ', ' . + $this->pages + ); + + self::assertEquals($expected, $bookSection['tx_lisztcommon_footer']); + } + + /** + * @test + */ + public function exampleEntryIsIndexedCorrectly(): void + { + $article = $this->subject->process($this->exampleArticleArray, new Collection(), new Collection()); + $expected = Str::of( + $this->bookTitle . ' ' . + $this->volume . + ' (' . $this->date . '), Nr. ' . + $this->issue . ', ' . + $this->pages + ); + + self::assertEquals($expected, $article['tx_lisztcommon_footer']); + } + + +} + +class SomeClass { public function doSomething($a) { return 'a';}} diff --git a/composer.json b/composer.json index 72c4aee..6bb3931 100644 --- a/composer.json +++ b/composer.json @@ -21,7 +21,8 @@ }, "autoload": { "psr-4": { - "Slub\\LisztBibliography\\": "Classes/" + "Slub\\LisztBibliography\\": "Classes/", + "Slub\\LisztBibliography\\Tests\\": "Tests/" } }, "extra": {