From 9eed9460efec4d6d0fb66c0d98560803b578ec61 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 27 Nov 2024 15:16:36 +0100 Subject: [PATCH 01/29] Test one part authors --- Classes/Processing/BibEntryProcessor.php | 25 ++++++++++--------- .../Unit/Processing/BibEntryProcessorTest.php | 21 ++++++++++++++++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/Classes/Processing/BibEntryProcessor.php b/Classes/Processing/BibEntryProcessor.php index b062028..170d862 100644 --- a/Classes/Processing/BibEntryProcessor.php +++ b/Classes/Processing/BibEntryProcessor.php @@ -16,8 +16,9 @@ use Illuminate\Support\Stringable; use Illuminate\Support\Str; use Slub\LisztCommon\Common\Collection; +use Slub\LisztCommon\Processing\IndexProcessor; -class BibEntryProcessor +class BibEntryProcessor extends IndexProcessor { public static function process( @@ -32,28 +33,28 @@ public static function process( $bibliographyItem['localizedCitations'][$locale] = $localizedCitation->get($key)['citation']; } $bibliographyItem['tei'] = $teiDataSets->get($key); - $bibliographyItem['tx_lisztcommon_header'] = self::buildListingField($bibliographyItem, BibEntryConfig::getAuthorHeader()); - if ($bibliographyItem['tx_lisztcommon_header'] == '') { - $bibliographyItem['tx_lisztcommon_header'] = self::buildListingField($bibliographyItem, BibEntryConfig::getEditorHeader()); + $bibliographyItem[self::HEADER_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::getAuthorHeader()); + if ($bibliographyItem[self::HEADER_FIELD] == '') { + $bibliographyItem[self::HEADER_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::getEditorHeader()); } - $bibliographyItem['tx_lisztcommon_body'] = self::buildListingField($bibliographyItem, BibEntryConfig::getBody()); - switch($bibliographyItem['itemType']) { + $bibliographyItem[self::BODY_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::getBody()); + switch($bibliographyItem[self::TYPE_FIELD]) { case 'book': - $bibliographyItem['tx_lisztcommon_footer'] = self::buildListingField($bibliographyItem, BibEntryConfig::getBookFooter()); + $bibliographyItem[self::FOOTER_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::getBookFooter()); break; case 'bookSection': - $bibliographyItem['tx_lisztcommon_footer'] = self::buildListingField($bibliographyItem, BibEntryConfig::getBookSectionFooter()); + $bibliographyItem[self::FOOTER_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::getBookSectionFooter()); break; case 'journalArticle': - $bibliographyItem['tx_lisztcommon_footer'] = self::buildListingField($bibliographyItem, BibEntryConfig::getArticleFooter()); + $bibliographyItem[self::FOOTER_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::getArticleFooter()); break; case 'thesis': - $bibliographyItem['tx_lisztcommon_footer'] = self::buildListingField($bibliographyItem, BibEntryConfig::getThesisFooter()); + $bibliographyItem[self::FOOTER_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::getThesisFooter()); break; } - $bibliographyItem['tx_lisztcommon_searchable'] = self::buildListingField($bibliographyItem, BibEntryConfig::SEARCHABLE_FIELDS); - $bibliographyItem['tx_lisztcommon_boosted'] = self::buildListingField($bibliographyItem, BibEntryConfig::BOOSTED_FIELDS); + $bibliographyItem[self::SEARCHABLE_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::SEARCHABLE_FIELDS); + $bibliographyItem[self::BOOSTED_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::BOOSTED_FIELDS); return $bibliographyItem; } diff --git a/Tests/Unit/Processing/BibEntryProcessorTest.php b/Tests/Unit/Processing/BibEntryProcessorTest.php index 0878387..6a05fb1 100644 --- a/Tests/Unit/Processing/BibEntryProcessorTest.php +++ b/Tests/Unit/Processing/BibEntryProcessorTest.php @@ -80,6 +80,24 @@ protected function setUp(): void } JSON; + $this->exampleBookWithAnonymousAuthor = + <<title", + "creators": [ + { + "creatorType": "editor", + "firstName": "", + "lastName": "$this->editorLastName" + } + ], + "place": "$this->place", + "date": "$this->date" + } + JSON; + $this->exampleArticle = <<subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); $article = $this->subject->process($this->exampleArticleArray, new Collection(), new Collection()); $bookWithoutAuthor = $this->subject->process($this->exampleBookWithoutAuthorArray, new Collection(), new Collection()); + $bookWithAnonymousAuthor = $this->subject->process($this->exampleBookWithoutAuthorArray, new Collection(), new Collection()); $expected = Str::of($this->authorFirstName . ' ' . $this->authorLastName); $expectedWithoutAuthor = Str::of($this->editorFirstName . ' ' . $this->editorLastName . ' (Hg.)'); + $expectedWithAnonymousAuthor = Str::of($this->authorLastName . ' (Hg.)'); self::assertEquals($book['tx_lisztcommon_header'], $expected); self::assertEquals($bookSection['tx_lisztcommon_header'], $expected); self::assertEquals($article['tx_lisztcommon_header'], $expected); self::assertEquals($bookWithoutAuthor['tx_lisztcommon_header'], $expectedWithoutAuthor); + self::assertEquals($bookWithAnonymousAuthor['tx_lisztcommon_header'], $expectedWithAnonymousAuthor); } /** From e3b8283c99192b292be4a2dd270db665d7d12cd1 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 27 Nov 2024 16:11:32 +0100 Subject: [PATCH 02/29] Add volume and number of volumes to footer --- Classes/Processing/BibEntryConfig.php | 9 ++- .../Unit/Processing/BibEntryProcessorTest.php | 57 +++++++++++++++++-- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/Classes/Processing/BibEntryConfig.php b/Classes/Processing/BibEntryConfig.php index bcf4c0c..6532ba5 100644 --- a/Classes/Processing/BibEntryConfig.php +++ b/Classes/Processing/BibEntryConfig.php @@ -124,6 +124,12 @@ class BibEntryConfig 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const NUMBER_OF_VOLUMES = [ + 'field' => 'numberOfVolumes', + 'conditionField' => 'pages', + 'conditionValue' => '', + 'conditionRelation' => 'neq' + ]; public static function getAuthorHeader(): array { @@ -155,7 +161,8 @@ public static function getBookSectionFooter(): array { return [ self::circumfix(self::BOOK_TITLE, 'In ', ', '), - self::postfix(self::VOLUME, ', '), + self::circumfix(self::VOLUME, 'Bd. ', ', '), + self::postfix(self::NUMBER_OF_VOLUMES, 'Bde., '), self::circumfix(self::EDITOR, 'hg. von ', ', '), self::circumfix(self::TRANSLATOR, 'übers. von ', ', '), self::postfix(self::PLACE, ' '), diff --git a/Tests/Unit/Processing/BibEntryProcessorTest.php b/Tests/Unit/Processing/BibEntryProcessorTest.php index 0878387..cdcbe36 100644 --- a/Tests/Unit/Processing/BibEntryProcessorTest.php +++ b/Tests/Unit/Processing/BibEntryProcessorTest.php @@ -27,12 +27,15 @@ final class BibEntryProcessorTest extends UnitTestCase private string $authorLastName = 'ex_author_last'; private string $editorFirstName = 'ex_editor_first'; private string $editorLastName = 'ex_editor_last'; + private string $translatorFirstName = 'ex_translator_first'; + private string $translatorLastName = 'ex_translator_last'; private string $title = 'ex_title'; private string $bookTitle = 'ex_book_title'; private string $place = 'ex_place'; private string $date = 'ex_date'; private string $pages = 'ex_pages'; private string $volume = 'ex_volume'; + private string $numberOfVolumes = 'ex_number_of_volumes'; private string $issue = 'ex_issue'; private string $exampleBook = ''; @@ -55,9 +58,21 @@ protected function setUp(): void "creatorType": "author", "firstName": "$this->authorFirstName", "lastName": "$this->authorLastName" + }, + { + "creatorType": "editor", + "firstName": "$this->editorFirstName", + "lastName": "$this->editorLastName" + }, + { + "creatorType": "translator", + "firstName": "$this->translatorFirstName", + "lastName": "$this->translatorLastName" } ], "place": "$this->place", + "volume": "$this->volume", + "numberOfVolumes": "$this->numberOfVolumes", "date": "$this->date" } JSON; @@ -73,8 +88,15 @@ protected function setUp(): void "creatorType": "editor", "firstName": "$this->editorFirstName", "lastName": "$this->editorLastName" + }, + { + "creatorType": "translator", + "firstName": "$this->translatorFirstName", + "lastName": "$this->translatorLastName" } ], + "volume": "$this->volume", + "numberOfVolumes": "$this->numberOfVolumes", "place": "$this->place", "date": "$this->date" } @@ -117,9 +139,16 @@ protected function setUp(): void "creatorType": "editor", "firstName": "$this->editorFirstName", "lastName": "$this->editorLastName" + }, + { + "creatorType": "translator", + "firstName": "$this->translatorFirstName", + "lastName": "$this->translatorLastName" } ], "bookTitle": "$this->bookTitle", + "volume": "$this->volume", + "numberOfVolumes": "$this->numberOfVolumes", "place": "$this->place", "date": "$this->date", "pages": "$this->pages" @@ -177,7 +206,13 @@ public function bodyIsProcessedCorrectly(): void public function bookFooterIsProcessedCorrectly(): void { $book = $this->subject->process($this->exampleBookArray, new Collection(), new Collection()); - $expected = Str::of($this->place . ' ' . $this->date); + $expected = Str::of( + 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . ', ' . + 'übers. von ' . $this->translatorFirstName . ' ' . $this->translatorLastName . ', ' . + 'Bd. ' . $this->volume . ', ' . + $this->place . ' ' . + $this->date + ); self::assertEquals($book['tx_lisztcommon_footer'], $expected); } @@ -188,8 +223,15 @@ public function bookFooterIsProcessedCorrectly(): void public function bookSectionFooterIsProcessedCorrectly(): void { $bookSection = $this->subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); - $expected = Str::of('In ' . $this->bookTitle . ', ' . 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . - ', ' . $this->place . ' ' . $this->date . ', ' . $this->pages); + $expected = Str::of( + 'In ' . $this->bookTitle . ', ' . + 'Bd. ' . $this->volume . ', ' . + 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . ', ' . + 'übers. von ' . $this->translatorFirstName . ' ' . $this->translatorLastName . ', ' . + $this->place . ' ' . + $this->date . ', ' . + $this->pages + ); self::assertEquals($bookSection['tx_lisztcommon_footer'], $expected); } @@ -200,8 +242,13 @@ public function bookSectionFooterIsProcessedCorrectly(): void public function articleFooterIsProcessedCorrectly(): void { $article = $this->subject->process($this->exampleArticleArray, new Collection(), new Collection()); - $expected = Str::of($this->bookTitle . ' ' . $this->volume . ' (' . $this->date . '), Nr. ' . $this->issue . ', ' . $this->pages); - var_dump($article['tx_lisztcommon_footer']); + $expected = Str::of( + $this->bookTitle . ' ' . + $this->volume . + ' (' . $this->date . '), Nr. ' . + $this->issue . ', ' . + $this->pages + ); self::assertEquals($article['tx_lisztcommon_footer'], $expected); } From 16242a8ca3fc358b274ca57c7d06c814dd9af120 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Thu, 28 Nov 2024 13:22:32 +0100 Subject: [PATCH 03/29] Fixes --- Tests/Unit/Processing/BibEntryProcessorTest.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Tests/Unit/Processing/BibEntryProcessorTest.php b/Tests/Unit/Processing/BibEntryProcessorTest.php index 6a05fb1..47b0d8f 100644 --- a/Tests/Unit/Processing/BibEntryProcessorTest.php +++ b/Tests/Unit/Processing/BibEntryProcessorTest.php @@ -20,6 +20,7 @@ final class BibEntryProcessorTest extends UnitTestCase private ?BibEntryProcessor $subject = null; private array $exampleBookArray = []; private array $exampleBookWithoutAuthorArray = []; + private array $exampleBookWithAnonymousAuthorArray = []; private array $exampleBookSectionArray = []; private array $exampleArticleArray = []; @@ -37,6 +38,7 @@ final class BibEntryProcessorTest extends UnitTestCase private string $exampleBook = ''; private string $exampleBookWithoutAuthor = ''; + private string $exampleBookWithAnonymousAuthor = ''; private string $exampleArticle = ''; private string $exampleBookSection = ''; @@ -88,9 +90,9 @@ protected function setUp(): void "title": "$this->title", "creators": [ { - "creatorType": "editor", + "creatorType": "author", "firstName": "", - "lastName": "$this->editorLastName" + "lastName": "$this->authorLastName" } ], "place": "$this->place", @@ -147,6 +149,7 @@ protected function setUp(): void $this->subject = GeneralUtility::makeInstance(BibEntryProcessor::class); $this->exampleBookArray = json_decode($this->exampleBook, true); $this->exampleBookWithoutAuthorArray = json_decode($this->exampleBookWithoutAuthor, true); + $this->exampleBookWithAnonymousAuthorArray = json_decode($this->exampleBookWithAnonymousAuthor, true); $this->exampleArticleArray = json_decode($this->exampleArticle, true); $this->exampleBookSectionArray = json_decode($this->exampleBookSection, true); } @@ -165,11 +168,11 @@ public function headerIsProcessedCorrectly(): void $bookSection = $this->subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); $article = $this->subject->process($this->exampleArticleArray, new Collection(), new Collection()); $bookWithoutAuthor = $this->subject->process($this->exampleBookWithoutAuthorArray, new Collection(), new Collection()); - $bookWithAnonymousAuthor = $this->subject->process($this->exampleBookWithoutAuthorArray, new Collection(), new Collection()); + $bookWithAnonymousAuthor = $this->subject->process($this->exampleBookWithAnonymousAuthorArray, new Collection(), new Collection()); $expected = Str::of($this->authorFirstName . ' ' . $this->authorLastName); $expectedWithoutAuthor = Str::of($this->editorFirstName . ' ' . $this->editorLastName . ' (Hg.)'); - $expectedWithAnonymousAuthor = Str::of($this->authorLastName . ' (Hg.)'); + $expectedWithAnonymousAuthor = Str::of($this->authorLastName); self::assertEquals($book['tx_lisztcommon_header'], $expected); self::assertEquals($bookSection['tx_lisztcommon_header'], $expected); From caf9f4e3ba8217d79e5b443eaa9306bd8019f5a8 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Thu, 28 Nov 2024 13:35:36 +0100 Subject: [PATCH 04/29] Fix tests --- Classes/Processing/BibEntryConfig.php | 10 ++++++---- Tests/Unit/Processing/BibEntryProcessorTest.php | 16 +++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Classes/Processing/BibEntryConfig.php b/Classes/Processing/BibEntryConfig.php index 6532ba5..33a84d5 100644 --- a/Classes/Processing/BibEntryConfig.php +++ b/Classes/Processing/BibEntryConfig.php @@ -126,9 +126,9 @@ class BibEntryConfig ]; const NUMBER_OF_VOLUMES = [ 'field' => 'numberOfVolumes', - 'conditionField' => 'pages', + 'conditionField' => 'numberOfVolumes', 'conditionValue' => '', - 'conditionRelation' => 'neq' + 'conditionRelation' => 'neq', ]; public static function getAuthorHeader(): array @@ -161,10 +161,10 @@ public static function getBookSectionFooter(): array { return [ self::circumfix(self::BOOK_TITLE, 'In ', ', '), - self::circumfix(self::VOLUME, 'Bd. ', ', '), - self::postfix(self::NUMBER_OF_VOLUMES, 'Bde., '), self::circumfix(self::EDITOR, 'hg. von ', ', '), self::circumfix(self::TRANSLATOR, 'übers. von ', ', '), + self::postfix(self::NUMBER_OF_VOLUMES, 'Bde., '), + self::circumfix(self::VOLUME, 'Bd. ', ', '), self::postfix(self::PLACE, ' '), self::postfix(self::DATE, ', '), self::PAGES @@ -176,6 +176,8 @@ public static function getBookFooter(): array return [ self::circumfix(self::EDITOR, 'hg. von ', ', '), self::circumfix(self::TRANSLATOR, 'übers. von ', ', '), + self::postfix(self::NUMBER_OF_VOLUMES, 'Bde., '), + self::circumfix(self::VOLUME, 'Bd. ', ', '), self::postfix(self::PLACE, ' '), self::DATE ]; diff --git a/Tests/Unit/Processing/BibEntryProcessorTest.php b/Tests/Unit/Processing/BibEntryProcessorTest.php index 6fb1368..08f7437 100644 --- a/Tests/Unit/Processing/BibEntryProcessorTest.php +++ b/Tests/Unit/Processing/BibEntryProcessorTest.php @@ -219,9 +219,9 @@ public function bodyIsProcessedCorrectly(): void $bookSection = $this->subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); $article = $this->subject->process($this->exampleArticleArray, new Collection(), new Collection()); - self::assertEquals($book['tx_lisztcommon_body'], Str::of($this->title)); - self::assertEquals($bookSection['tx_lisztcommon_body'], Str::of($this->title)); - self::assertEquals($article['tx_lisztcommon_body'], Str::of($this->title)); + self::assertEquals(Str::of($this->title), $book['tx_lisztcommon_body']); + self::assertEquals(Str::of($this->title), $bookSection['tx_lisztcommon_body']); + self::assertEquals(Str::of($this->title), $article['tx_lisztcommon_body']); } /** @@ -233,12 +233,13 @@ public function bookFooterIsProcessedCorrectly(): void $expected = Str::of( 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . ', ' . 'übers. von ' . $this->translatorFirstName . ' ' . $this->translatorLastName . ', ' . + $this->numberOfVolumes . 'Bde., ' . 'Bd. ' . $this->volume . ', ' . $this->place . ' ' . $this->date ); - self::assertEquals($book['tx_lisztcommon_footer'], $expected); + self::assertEquals($expected, $book['tx_lisztcommon_footer']); } /** @@ -249,15 +250,16 @@ public function bookSectionFooterIsProcessedCorrectly(): void $bookSection = $this->subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); $expected = Str::of( 'In ' . $this->bookTitle . ', ' . - 'Bd. ' . $this->volume . ', ' . 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . ', ' . 'übers. von ' . $this->translatorFirstName . ' ' . $this->translatorLastName . ', ' . + $this->numberOfVolumes . 'Bde., ' . + 'Bd. ' . $this->volume . ', ' . $this->place . ' ' . $this->date . ', ' . $this->pages ); - self::assertEquals($bookSection['tx_lisztcommon_footer'], $expected); + self::assertEquals($expected, $bookSection['tx_lisztcommon_footer']); } /** @@ -274,7 +276,7 @@ public function articleFooterIsProcessedCorrectly(): void $this->pages ); - self::assertEquals($article['tx_lisztcommon_footer'], $expected); + self::assertEquals($expected, $article['tx_lisztcommon_footer']); } From c069a40527730f0336be0d375f8feca366087d9a Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Tue, 3 Dec 2024 17:44:05 +0100 Subject: [PATCH 05/29] add alias handling and atomic swap and remove of old indexes --- Classes/Command/IndexCommand.php | 163 ++++++++++++++++++++++++++----- 1 file changed, 140 insertions(+), 23 deletions(-) diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index a05e4dc..e357435 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -143,33 +143,46 @@ protected function fullSync(InputInterface $input): void $collection = new Collection($response->getBody()); $this->bibliographyItems = $collection->pluck('data'); $cursor = 0; // set Cursor to 0, not to bulk size - $index = $this->extConf['elasticIndexName']; - $mappingParams = BibElasticMapping::getMappingParams($index); + // we are working with alias names to swap indexes from zotero_temp to zotero after successfully indexing + $tempIndexAlias = $this->extConf['elasticIndexName'].'_temp'; + $indexName = $this->extConf['elasticIndexName'] . '_' . date('Ymd_His'); + $tempIndexParams = BibElasticMapping::getMappingParams($indexName); + + // add alias name 'zotero_temp to this index + // and add a wildcard alias to find all zotero_* indices with the alias zotero-index + $aliasParams = [ + 'body' => [ + 'actions' => [ + [ + 'add' => [ + 'index' => $indexName, + 'alias' => $tempIndexAlias, + ], + ], + [ + 'add' => [ + 'index' => $this->extConf['elasticIndexName'].'_*', + 'alias' => $this->extConf['elasticIndexName'].'-index', + ], + ] + ] + ] + ]; try { - // in older Elasticsearch versions (until 7) exists returns a bool - if ($this->client->indices()->exists(['index' => $index])) { - $this->client->indices()->delete(['index' => $index]); - $this->client->indices()->create($mappingParams); - } + $this->client->indices()->create($tempIndexParams); + $this->client->indices()->updateAliases($aliasParams); } catch (\Exception $e) { - // other versions return a Message object - if ($e->getCode() === 404) { - $this->io->note("Index: " . $index . " does not exist. Trying to create new index."); - $this->client->indices()->create($mappingParams); - } else { $this->io->error("Exception: " . $e->getMessage()); $this->logger->error('Bibliography sync unsuccessful. Error creating elasticsearch index.'); throw new \Exception('Bibliography sync unsuccessful.'); - } } $apiCounter = self::API_TRIALS; while ($cursor < $this->total) { try { - $this->sync($cursor, 0); - + $this->sync($indexName, $cursor, 0); $apiCounter = self::API_TRIALS; $remainingItems = $this->total - $cursor; $advanceBy = min($remainingItems, $this->bulkSize); @@ -188,6 +201,11 @@ protected function fullSync(InputInterface $input): void } } } + + // swap alias for index from zotero_temp to zotero and remove old indexes (keep the last one) + $this->swapIndexAliases($indexName, $tempIndexAlias); + //delete old indexes + $this->deleteOldIndexes($indexName); $this->io->progressFinish(); } @@ -196,7 +214,7 @@ protected function versionedSync(int $version): void $apiCounter = self::API_TRIALS; while (true) { try { - $this->sync(0, $version); + $this->sync( $this->extConf['elasticIndexName'], 0, $version); $this->io->text('done'); return; } catch (\Exception $e) { @@ -214,13 +232,13 @@ protected function versionedSync(int $version): void } } - protected function sync(int $cursor = 0, int $version = 0): void + protected function sync(string $indexName, int $cursor = 0, int $version = 0,): void { $this->fetchBibliography($cursor, $version); $this->fetchCitations($cursor, $version); $this->fetchTeiData($cursor, $version); $this->buildDataSets(); - $this->commitBibliography(); + $this->commitBibliography($indexName); } protected function getVersion(InputInterface $input): int @@ -344,21 +362,18 @@ protected function buildDataSets(): void }); } - protected function commitBibliography(): void + protected function commitBibliography(string $indexName): void { if ($this->dataSets->count() == 0) { $this->io->text('no new bibliographic entries'); return; } - $index = $this->extConf['elasticIndexName']; - $params = [ 'body' => [] ]; - $bulkCount = 0; foreach ($this->dataSets as $document) { $params['body'][] = [ 'index' => [ - '_index' => $index, + '_index' => $indexName, '_id' => $document['key'] ] ]; @@ -372,6 +387,108 @@ protected function commitBibliography(): void $this->client->bulk($params); } + protected function swapIndexAliases(string $indexName, string $tempIndexAlias): void + { + // get index with alias = zotero + try { + $aliasesRequest = $this->client->indices()->getAlias(['name' => $this->extConf['elasticIndexName']]); + $aliasesArray = $aliasesRequest->asArray(); + + foreach ($aliasesArray as $index => $aliasArray) { + $this->io->note('Remove alias "' .$this->extConf['elasticIndexName']. '" from index '. $index . 'and add it to ' . $indexName ); + // get index name with alias 'zotero' + if (array_key_exists($this->extConf['elasticIndexName'], $aliasArray['aliases'])) { + //swap alias from old to new index + $aliasParams = [ + 'body' => [ + 'actions' => [ + [ + 'remove' => [ + 'index' => $index, + 'alias' => $this->extConf['elasticIndexName'], + ], + ], + [ + 'add' => [ + 'index' => $indexName, + 'alias' => $this->extConf['elasticIndexName'], + ], + ] + ] + ] + ]; + $this->client->indices()->updateAliases($aliasParams); + } + } + } + catch (\Exception $e) { + // other versions return a Message object + if ($e->getCode() === 404) { + $this->io->note("Alias: " . $this->extConf['elasticIndexName'] . " does not exist. Move alias to ".$indexName); + // rename alias name from temp index to zotero + $aliasParams = [ + 'body' => [ + 'actions' => [ + [ + 'remove' => [ + 'index' => $indexName, + 'alias' => $tempIndexAlias, + ], + ], + [ + 'add' => [ + 'index' => $indexName, + 'alias' => $this->extConf['elasticIndexName'], + ], + ] + ] + ] + ]; + $this->client->indices()->updateAliases($aliasParams); + + } else { + $this->io->error("Exception: " . $e->getMessage()); + $this->logger->error('Bibliography sync unsuccessful. Error getting alias: ' . $this->extConf['elasticIndexName']); + throw new \Exception('Bibliography sync unsuccessful.', 0, $e); + } + } + } + + protected function deleteOldIndexes($indexName): void + { + try { + $aliasesRequest = $this->client->indices()->getAlias(['name' => $this->extConf['elasticIndexName'].'_*']); + $aliasesArray = $aliasesRequest->asArray(); + + // sort $aliasesArray by key name + ksort($aliasesArray); + + // remove current key $indexName from array + unset($aliasesArray[$indexName]); + + // remove the last key (we keep the last two indexes) + array_pop($aliasesArray); + + foreach ($aliasesArray as $index => $aliasArray) { + $this->io->note("Delete index " . $index); + $this->client->indices()->delete(['index' => $index]); + } + + } + catch (\Exception $e) { + // other versions return a Message object + if ($e->getCode() === 404) { + $this->io->note("Nothing to remove, there are no indexes with alias " . $this->extConf['elasticIndexName'].'_*'); + } else { + $this->io->error("Exception: " . $e->getMessage()); + $this->logger->error('Bibliography sync unsuccessful. Error getting alias: ' . $this->extConf['elasticIndexName'].'_*'); + throw new \Exception('Bibliography sync unsuccessful.', 0, $e); + } + } + + + } + /* protected function commitLocales(): void { $localeIndex = $this->extConf['elasticLocaleIndexName']; From 001ee8b79f93cfa8a1062ca079e85396a2de21c3 Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Wed, 4 Dec 2024 09:42:22 +0100 Subject: [PATCH 06/29] change selection of indexes to delete from wildcard alias to "zotero-index" alias --- Classes/Command/IndexCommand.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index e357435..0488941 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -457,7 +457,7 @@ protected function swapIndexAliases(string $indexName, string $tempIndexAlias): protected function deleteOldIndexes($indexName): void { try { - $aliasesRequest = $this->client->indices()->getAlias(['name' => $this->extConf['elasticIndexName'].'_*']); + $aliasesRequest = $this->client->indices()->getAlias(['name' => $this->extConf['elasticIndexName'].'-index']); $aliasesArray = $aliasesRequest->asArray(); // sort $aliasesArray by key name @@ -478,10 +478,10 @@ protected function deleteOldIndexes($indexName): void catch (\Exception $e) { // other versions return a Message object if ($e->getCode() === 404) { - $this->io->note("Nothing to remove, there are no indexes with alias " . $this->extConf['elasticIndexName'].'_*'); + $this->io->note("Nothing to remove, there are no indexes with alias " . $this->extConf['elasticIndexName'].'-index'); } else { $this->io->error("Exception: " . $e->getMessage()); - $this->logger->error('Bibliography sync unsuccessful. Error getting alias: ' . $this->extConf['elasticIndexName'].'_*'); + $this->logger->error('Bibliography sync unsuccessful. Error getting alias: ' . $this->extConf['elasticIndexName'].'-index'); throw new \Exception('Bibliography sync unsuccessful.', 0, $e); } } From 117161035cbd4eb90efa990ad1692d9292a04989 Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Wed, 4 Dec 2024 10:35:24 +0100 Subject: [PATCH 07/29] fix remove _temp Alias on old indexes --- Classes/Command/IndexCommand.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index 0488941..f59310e 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -413,6 +413,12 @@ protected function swapIndexAliases(string $indexName, string $tempIndexAlias): 'index' => $indexName, 'alias' => $this->extConf['elasticIndexName'], ], + ], + [ + 'remove' => [ + 'index' => $indexName, + 'alias' => $tempIndexAlias, + ], ] ] ] From 1e385fb143988be6dfd6ac3eabc9caaff37b55c2 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 4 Dec 2024 11:22:48 +0100 Subject: [PATCH 08/29] Promote indexName to property --- Classes/Command/IndexCommand.php | 51 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index f59310e..491b3db 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -37,21 +37,22 @@ class IndexCommand extends Command { - protected ZoteroApi $bibApi; const API_TRIALS = 3; protected string $apiKey; + protected ZoteroApi $bibApi; protected Collection $bibliographyItems; - protected Collection $deletedItems; - protected Collection $teiDataSets; - protected Collection $dataSets; + protected int $bulkSize; protected Client $client; + protected Collection $dataSets; + protected Collection $deletedItems; protected array $extConf; + readonly string $indexName; protected SymfonyStyle $io; - protected int $bulkSize; - protected int $total; protected Collection $locales; protected Collection $localizedCitations; + protected Collection $teiDataSets; + protected int $total; public function __construct( private readonly SiteFinder $siteFinder, @@ -101,6 +102,7 @@ protected function configure(): void protected function initialize(InputInterface $input, OutputInterface $output): void { $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography'); + $this->indexName = $this->extConf['elasticIndexName'] . '_' . date('Ymd_His'); $this->client = ElasticClientBuilder::getClient(); $this->apiKey = $this->extConf['zoteroApiKey']; $this->io = new SymfonyStyle($input, $output); @@ -145,8 +147,7 @@ protected function fullSync(InputInterface $input): void $cursor = 0; // set Cursor to 0, not to bulk size // we are working with alias names to swap indexes from zotero_temp to zotero after successfully indexing $tempIndexAlias = $this->extConf['elasticIndexName'].'_temp'; - $indexName = $this->extConf['elasticIndexName'] . '_' . date('Ymd_His'); - $tempIndexParams = BibElasticMapping::getMappingParams($indexName); + $tempIndexParams = BibElasticMapping::getMappingParams($this->indexName); // add alias name 'zotero_temp to this index // and add a wildcard alias to find all zotero_* indices with the alias zotero-index @@ -155,7 +156,7 @@ protected function fullSync(InputInterface $input): void 'actions' => [ [ 'add' => [ - 'index' => $indexName, + 'index' => $this->indexName, 'alias' => $tempIndexAlias, ], ], @@ -182,7 +183,7 @@ protected function fullSync(InputInterface $input): void while ($cursor < $this->total) { try { - $this->sync($indexName, $cursor, 0); + $this->sync($cursor, 0); $apiCounter = self::API_TRIALS; $remainingItems = $this->total - $cursor; $advanceBy = min($remainingItems, $this->bulkSize); @@ -203,9 +204,9 @@ protected function fullSync(InputInterface $input): void } // swap alias for index from zotero_temp to zotero and remove old indexes (keep the last one) - $this->swapIndexAliases($indexName, $tempIndexAlias); + $this->swapIndexAliases($tempIndexAlias); //delete old indexes - $this->deleteOldIndexes($indexName); + $this->deleteOldIndexes(); $this->io->progressFinish(); } @@ -214,7 +215,7 @@ protected function versionedSync(int $version): void $apiCounter = self::API_TRIALS; while (true) { try { - $this->sync( $this->extConf['elasticIndexName'], 0, $version); + $this->sync(0, $version); $this->io->text('done'); return; } catch (\Exception $e) { @@ -232,13 +233,13 @@ protected function versionedSync(int $version): void } } - protected function sync(string $indexName, int $cursor = 0, int $version = 0,): void + protected function sync(int $cursor = 0, int $version = 0,): void { $this->fetchBibliography($cursor, $version); $this->fetchCitations($cursor, $version); $this->fetchTeiData($cursor, $version); $this->buildDataSets(); - $this->commitBibliography($indexName); + $this->commitBibliography(); } protected function getVersion(InputInterface $input): int @@ -362,7 +363,7 @@ protected function buildDataSets(): void }); } - protected function commitBibliography(string $indexName): void + protected function commitBibliography(): void { if ($this->dataSets->count() == 0) { $this->io->text('no new bibliographic entries'); @@ -373,7 +374,7 @@ protected function commitBibliography(string $indexName): void foreach ($this->dataSets as $document) { $params['body'][] = [ 'index' => [ - '_index' => $indexName, + '_index' => $this->indexName, '_id' => $document['key'] ] ]; @@ -387,7 +388,7 @@ protected function commitBibliography(string $indexName): void $this->client->bulk($params); } - protected function swapIndexAliases(string $indexName, string $tempIndexAlias): void + protected function swapIndexAliases(string $tempIndexAlias): void { // get index with alias = zotero try { @@ -395,7 +396,7 @@ protected function swapIndexAliases(string $indexName, string $tempIndexAlias): $aliasesArray = $aliasesRequest->asArray(); foreach ($aliasesArray as $index => $aliasArray) { - $this->io->note('Remove alias "' .$this->extConf['elasticIndexName']. '" from index '. $index . 'and add it to ' . $indexName ); + $this->io->note('Remove alias "' .$this->extConf['elasticIndexName']. '" from index '. $index . ' and add it to ' . $this->indexName ); // get index name with alias 'zotero' if (array_key_exists($this->extConf['elasticIndexName'], $aliasArray['aliases'])) { //swap alias from old to new index @@ -410,7 +411,7 @@ protected function swapIndexAliases(string $indexName, string $tempIndexAlias): ], [ 'add' => [ - 'index' => $indexName, + 'index' => $this->indexName, 'alias' => $this->extConf['elasticIndexName'], ], ], @@ -430,20 +431,20 @@ protected function swapIndexAliases(string $indexName, string $tempIndexAlias): catch (\Exception $e) { // other versions return a Message object if ($e->getCode() === 404) { - $this->io->note("Alias: " . $this->extConf['elasticIndexName'] . " does not exist. Move alias to ".$indexName); + $this->io->note("Alias: " . $this->extConf['elasticIndexName'] . " does not exist. Move alias to ".$this->indexName); // rename alias name from temp index to zotero $aliasParams = [ 'body' => [ 'actions' => [ [ 'remove' => [ - 'index' => $indexName, + 'index' => $this->indexName, 'alias' => $tempIndexAlias, ], ], [ 'add' => [ - 'index' => $indexName, + 'index' => $this->indexName, 'alias' => $this->extConf['elasticIndexName'], ], ] @@ -460,7 +461,7 @@ protected function swapIndexAliases(string $indexName, string $tempIndexAlias): } } - protected function deleteOldIndexes($indexName): void + protected function deleteOldIndexes(): void { try { $aliasesRequest = $this->client->indices()->getAlias(['name' => $this->extConf['elasticIndexName'].'-index']); @@ -470,7 +471,7 @@ protected function deleteOldIndexes($indexName): void ksort($aliasesArray); // remove current key $indexName from array - unset($aliasesArray[$indexName]); + unset($aliasesArray[$this->indexName]); // remove the last key (we keep the last two indexes) array_pop($aliasesArray); From ea96685ad7b034ad92f64d9f2021de135d15a873 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 4 Dec 2024 15:45:19 +0100 Subject: [PATCH 09/29] Promote several variables to properties --- Classes/Command/IndexCommand.php | 40 +++++++++++++++++--------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index 491b3db..50cf66b 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -39,8 +39,7 @@ class IndexCommand extends Command const API_TRIALS = 3; - protected string $apiKey; - protected ZoteroApi $bibApi; + protected string $zoteroApiKey; protected Collection $bibliographyItems; protected int $bulkSize; protected Client $client; @@ -48,9 +47,11 @@ class IndexCommand extends Command protected Collection $deletedItems; protected array $extConf; readonly string $indexName; + protected InputInterface $input; protected SymfonyStyle $io; protected Collection $locales; protected Collection $localizedCitations; + protected OutputInterface $output; protected Collection $teiDataSets; protected int $total; @@ -101,21 +102,23 @@ protected function configure(): void protected function initialize(InputInterface $input, OutputInterface $output): void { + $this->input = $input; + $this->output = $output; $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography'); $this->indexName = $this->extConf['elasticIndexName'] . '_' . date('Ymd_His'); $this->client = ElasticClientBuilder::getClient(); - $this->apiKey = $this->extConf['zoteroApiKey']; - $this->io = new SymfonyStyle($input, $output); + $this->zoteroApiKey = $this->extConf['zoteroApiKey']; + $this->io = GeneralUtility::makeInstance(SymfonyStyle::class, $this->input, $this->output); $this->io->title($this->getDescription()); } protected function execute(InputInterface $input, OutputInterface $output): int { $this->bulkSize = (int) $this->extConf['zoteroBulkSize']; - $version = $this->getVersion($input); + $version = $this->getVersion(); if ($version == 0) { $this->io->text('Full data synchronization requested.'); - $this->fullSync($input); + $this->fullSync(); $this->logger->info('Full data synchronization successful.'); } else { $this->io->text('Synchronizing all data from version ' . $version); @@ -125,17 +128,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::SUCCESS; } - protected function fullSync(InputInterface $input): void + protected function fullSync(): void { - $client = new ZoteroApi($this->extConf['zoteroApiKey']); + $client = GeneralUtility::makeInstance(ZoteroApi::class, $this->zoteroApiKey); $response = $client-> group($this->extConf['zoteroGroupId'])-> items()-> top()-> limit(1)-> send(); - if ($input->getOption('total')) { - $this->total = (int) $input->getOption('total'); + if ($this->input->getOption('total')) { + $this->total = (int) $this->input->getOption('total'); } else { $this->total = (int) $response->getHeaders()['Total-Results'][0]; } @@ -233,7 +236,7 @@ protected function versionedSync(int $version): void } } - protected function sync(int $cursor = 0, int $version = 0,): void + protected function sync(int $cursor = 0, int $version = 0): void { $this->fetchBibliography($cursor, $version); $this->fetchCitations($cursor, $version); @@ -242,22 +245,22 @@ protected function sync(int $cursor = 0, int $version = 0,): void $this->commitBibliography(); } - protected function getVersion(InputInterface $input): int + protected function getVersion(): int { // if -a is specified, perfom a full update - if ($input->getOption('all')) { + if ($this->input->getOption('all')) { return 0; } // also set version to 0 for dev tests if the total results are limited - if ($input->getOption('total')) { - $this->io->text('Total results limited to: '. $input->getOption('total')); + if ($this->input->getOption('total')) { + $this->io->text('Total results limited to: '. $this->input->getOption('total')); return 0; } // if a version is manually specified, perform sync from this version - $argumentVersion = $input->getArgument('version'); + $argumentVersion = $this->input->getArgument('version'); if ($argumentVersion > 0) { return (int) $argumentVersion; } @@ -294,7 +297,7 @@ protected function getVersion(InputInterface $input): int protected function fetchBibliography(int $cursor, int $version): void { - $client = new ZoteroApi($this->extConf['zoteroApiKey']); + $client = GeneralUtility::makeInstance(ZoteroApi::class, $this->zoteroApiKey); $response = $client-> group($this->extConf['zoteroGroupId'])-> items()-> @@ -310,7 +313,6 @@ protected function fetchBibliography(int $cursor, int $version): void protected function fetchCitations(int $cursor, int $version): void { - $this->localizedCitations = new Collection(); $this->locales->each(function($locale) use($cursor, $version) { $this->fetchCitationLocale($locale, $cursor, $version); }); } @@ -339,7 +341,7 @@ protected function fetchCitationLocale(string $locale, int $cursor, int $version protected function fetchTeiData(int $cursor, int $version): void { - $client = new ZoteroApi($this->extConf['zoteroApiKey']); + $client = GeneralUtility::makeInstance(ZoteroApi::class, $this->zoteroApiKey); $response = $client-> group($this->extConf['zoteroGroupId'])-> items()-> From 653712389338ca63ecb6cb32eaad21457d83aa33 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 4 Dec 2024 15:47:29 +0100 Subject: [PATCH 10/29] Fixes --- Classes/Command/IndexCommand.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index 50cf66b..3a9ab55 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -60,6 +60,8 @@ public function __construct( private readonly LoggerInterface $logger ) { parent::__construct(); + $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography'); + $this->indexName = $this->extConf['elasticIndexName'] . '_' . date('Ymd_His'); $this->initLocales(); } @@ -104,8 +106,6 @@ protected function initialize(InputInterface $input, OutputInterface $output): v { $this->input = $input; $this->output = $output; - $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography'); - $this->indexName = $this->extConf['elasticIndexName'] . '_' . date('Ymd_His'); $this->client = ElasticClientBuilder::getClient(); $this->zoteroApiKey = $this->extConf['zoteroApiKey']; $this->io = GeneralUtility::makeInstance(SymfonyStyle::class, $this->input, $this->output); @@ -419,7 +419,7 @@ protected function swapIndexAliases(string $tempIndexAlias): void ], [ 'remove' => [ - 'index' => $indexName, + 'index' => $this->indexName, 'alias' => $tempIndexAlias, ], ] From 28aaf7f38a731452e14b0fe41b2e2cf8491e735a Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Tue, 10 Dec 2024 18:29:54 +0100 Subject: [PATCH 11/29] Add colon and add in to journal article footer --- Classes/Processing/BibEntryConfig.php | 161 ++++++++++++++------------ 1 file changed, 87 insertions(+), 74 deletions(-) diff --git a/Classes/Processing/BibEntryConfig.php b/Classes/Processing/BibEntryConfig.php index 33a84d5..108d7f4 100644 --- a/Classes/Processing/BibEntryConfig.php +++ b/Classes/Processing/BibEntryConfig.php @@ -35,6 +35,7 @@ class BibEntryConfig 'separator' => ', ' ] ]; + const EDITOR = [ 'compound' => [ 'fields' => [ @@ -55,6 +56,7 @@ class BibEntryConfig 'separator' => ', ' ] ]; + const TRANSLATOR = [ 'compound' => [ 'fields' => [ @@ -75,55 +77,65 @@ class BibEntryConfig 'separator' => ', ' ] ]; + const TITLE = [ 'field' => 'title' ]; + const PUBLICATION_TITLE = [ 'field' => 'publicationTitle', 'conditionField' => 'publicationTitle', 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const BOOK_TITLE = [ 'field' => 'bookTitle', 'conditionField' => 'bookTitle', 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const UNIVERSITY = [ 'field' => 'university', 'conditionField' => 'university', 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const VOLUME = [ 'field' => 'volume', 'conditionField' => 'volume', 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const ISSUE = [ 'field' => 'issue', 'conditionField' => 'issue', 'conditionValue' => '', 'conditionRelation' => 'neq' ]; + const PLACE = [ 'field' => 'place', 'conditionField' => 'place', 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const DATE = [ 'field' => 'date', 'conditionField' => 'date', 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const PAGES = [ 'field' => 'pages', 'conditionField' => 'pages', 'conditionValue' => '', 'conditionRelation' => 'neq', ]; + const NUMBER_OF_VOLUMES = [ 'field' => 'numberOfVolumes', 'conditionField' => 'numberOfVolumes', @@ -131,6 +143,79 @@ class BibEntryConfig 'conditionRelation' => 'neq', ]; + const SEARCHABLE_FIELDS = [ + [ + 'compound' => [ + 'fields' => [ + [ + 'field' => 'firstName', + 'conditionField' => 'creatorType', + 'conditionValue' => 'author', + 'conditionRelation' => 'eq' + ], + [ + 'field' => 'lastName', + 'conditionField' => 'creatorType', + 'conditionValue' => 'author', + 'conditionRelation' => 'eq' + ] + ], + 'field' => 'creators', + 'separator' => ' ', + 'postfix' => ' ' + ] + ], + [ + 'field' => 'title', + 'postfix' => ' ' + ], + [ + 'field' => 'university', + 'postfix' => ' ' + ], + [ + 'field' => 'bookTitle', + 'postfix' => ' ' + ], + [ + 'field' => 'series', + 'postfix' => ' ' + ], + [ + 'field' => 'publicationTitle', + 'postfix' => ' ' + ], + [ + 'field' => 'place', + 'postfix' => ' ' + ], + [ + 'field' => 'date', + 'postfix' => ' ' + ] + ]; + + const BOOSTED_FIELDS = [ + [ + 'compound' => [ + 'fields' => [ + [ + 'field' => 'lastName', + 'conditionField' => 'creatorType', + 'conditionValue' => 'author', + 'conditionRelation' => 'eq' + ] + ], + 'field' => 'creators', + 'separator' => ' ', + 'reverseFirst' => true, + 'postfix' => ' ' + ] + ], + [ 'field' => 'title' ], + [ 'field' => 'date' ] + ]; + public static function getAuthorHeader(): array { return [ self::AUTHOR ]; @@ -149,7 +234,7 @@ public static function getBody(): array public static function getArticleFooter(): array { return [ - self::postfix(self::PUBLICATION_TITLE, ' '), + self::circumfix(self::PUBLICATION_TITLE, 'in: ', ' '), self::postfix(self::VOLUME, ' '), self::circumfix(self::DATE, '(', '), '), self::circumfix(self::ISSUE, 'Nr. ', ', '), @@ -160,7 +245,7 @@ public static function getArticleFooter(): array public static function getBookSectionFooter(): array { return [ - self::circumfix(self::BOOK_TITLE, 'In ', ', '), + self::circumfix(self::BOOK_TITLE, 'in: ', ', '), self::circumfix(self::EDITOR, 'hg. von ', ', '), self::circumfix(self::TRANSLATOR, 'übers. von ', ', '), self::postfix(self::NUMBER_OF_VOLUMES, 'Bde., '), @@ -236,76 +321,4 @@ private static function surroundComma(array $field): array { return self::comma(self::surround($field)); } - - const SEARCHABLE_FIELDS = [ - [ - 'compound' => [ - 'fields' => [ - [ - 'field' => 'firstName', - 'conditionField' => 'creatorType', - 'conditionValue' => 'author', - 'conditionRelation' => 'eq' - ], - [ - 'field' => 'lastName', - 'conditionField' => 'creatorType', - 'conditionValue' => 'author', - 'conditionRelation' => 'eq' - ] - ], - 'field' => 'creators', - 'separator' => ' ', - 'postfix' => ' ' - ] - ], - [ - 'field' => 'title', - 'postfix' => ' ' - ], - [ - 'field' => 'university', - 'postfix' => ' ' - ], - [ - 'field' => 'bookTitle', - 'postfix' => ' ' - ], - [ - 'field' => 'series', - 'postfix' => ' ' - ], - [ - 'field' => 'publicationTitle', - 'postfix' => ' ' - ], - [ - 'field' => 'place', - 'postfix' => ' ' - ], - [ - 'field' => 'date', - 'postfix' => ' ' - ] - ]; - const BOOSTED_FIELDS = [ - [ - 'compound' => [ - 'fields' => [ - [ - 'field' => 'lastName', - 'conditionField' => 'creatorType', - 'conditionValue' => 'author', - 'conditionRelation' => 'eq' - ] - ], - 'field' => 'creators', - 'separator' => ' ', - 'reverseFirst' => true, - 'postfix' => ' ' - ] - ], - [ 'field' => 'title' ], - [ 'field' => 'date' ] - ]; } From 40ae1824c97961d344355ec00541d77982159b3e Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Tue, 10 Dec 2024 20:58:33 +0100 Subject: [PATCH 12/29] Reorder facets and remove translators from creator facet --- Configuration/TypoScript/setup.typoscript | 28 +++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index ce4a293..208f40f 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -41,29 +41,33 @@ plugin.tx_lisztcommon_searchlisting { type = terms } 1 { - field = place - type = terms - } - 2 { - field = date - type = terms - } - 3 { - field = publicationTitle - type = terms - } - 4 { field = creators type = nested script ( String firstName = doc['creators.firstName.keyword'].size() > 0 ? doc['creators.firstName.keyword'].value : ''; String lastName = doc['creators.lastName.keyword'].size() > 0 ? doc['creators.lastName.keyword'].value : ''; + String type = doc['creators.creatorType.keyword'].size() > 0 ? doc['creators.creatorType.keyword'].value : ''; + if (type == 'translator') { + return null; + } if (firstName == '' && lastName == '') { return null; } return (firstName + ' ' + lastName).trim(); ) } + 2 { + field = date + type = terms + } + 3 { + field = language + type = terms + } + 4 { + field = publicationTitle + type = terms + } } } } From d8efe57d803d5e072ed492d8543324f6f6ae2f96 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Tue, 10 Dec 2024 21:58:56 +0100 Subject: [PATCH 13/29] Add facet fields --- Classes/Processing/BibElasticMapping.php | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Classes/Processing/BibElasticMapping.php b/Classes/Processing/BibElasticMapping.php index 8f897c4..724172e 100644 --- a/Classes/Processing/BibElasticMapping.php +++ b/Classes/Processing/BibElasticMapping.php @@ -15,22 +15,28 @@ public static function getMappingParams(string $index): array 'mappings' => [ 'dynamic' => false, 'properties' => [ + 'itemType' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'version' => [ 'type' => 'long' ], 'title' => [ 'type' => 'text'], 'university' => [ 'type' => 'text'], 'bookTitle' => [ 'type' => 'text'], 'series' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'publicationTitle' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], + 'language' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'place' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'date' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'archiveLocation' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], - 'itemType' => [ 'type' => 'keyword'], 'journalTitle' => [ 'type' => 'keyword'], 'creators' => [ 'type' => 'nested', 'properties' => [ 'creatorType' => [ - 'type' => 'keyword' + 'type' => 'text', + 'fields' => [ + 'keyword' => [ + 'type' => 'keyword', 'ignore_above' => 256 + ] + ] ], 'firstName' => [ 'type' => 'text', From ea8d019525cdf3fcba4d9f5735ebe38e900e5613 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Tue, 10 Dec 2024 22:02:28 +0100 Subject: [PATCH 14/29] Adjust test --- Tests/Unit/Processing/BibEntryProcessorTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/Unit/Processing/BibEntryProcessorTest.php b/Tests/Unit/Processing/BibEntryProcessorTest.php index 08f7437..68b2fff 100644 --- a/Tests/Unit/Processing/BibEntryProcessorTest.php +++ b/Tests/Unit/Processing/BibEntryProcessorTest.php @@ -249,7 +249,7 @@ public function bookSectionFooterIsProcessedCorrectly(): void { $bookSection = $this->subject->process($this->exampleBookSectionArray, new Collection(), new Collection()); $expected = Str::of( - 'In ' . $this->bookTitle . ', ' . + 'in: ' . $this->bookTitle . ', ' . 'hg. von ' . $this->editorFirstName . ' ' . $this->editorLastName . ', ' . 'übers. von ' . $this->translatorFirstName . ' ' . $this->translatorLastName . ', ' . $this->numberOfVolumes . 'Bde., ' . @@ -269,7 +269,7 @@ public function articleFooterIsProcessedCorrectly(): void { $article = $this->subject->process($this->exampleArticleArray, new Collection(), new Collection()); $expected = Str::of( - $this->bookTitle . ' ' . + 'in: ' . $this->bookTitle . ' ' . $this->volume . ' (' . $this->date . '), Nr. ' . $this->issue . ', ' . From 9b22676c021b00ff36198402f7c13bb9dd8bef15 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 11 Dec 2024 11:14:53 +0100 Subject: [PATCH 15/29] Adjust mapping --- Classes/Processing/BibElasticMapping.php | 11 +++-------- Configuration/TypoScript/setup.typoscript | 6 +++--- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Classes/Processing/BibElasticMapping.php b/Classes/Processing/BibElasticMapping.php index 724172e..c911930 100644 --- a/Classes/Processing/BibElasticMapping.php +++ b/Classes/Processing/BibElasticMapping.php @@ -15,14 +15,14 @@ public static function getMappingParams(string $index): array 'mappings' => [ 'dynamic' => false, 'properties' => [ - 'itemType' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], + 'itemType' => [ 'type' => 'keyword' ], 'version' => [ 'type' => 'long' ], 'title' => [ 'type' => 'text'], 'university' => [ 'type' => 'text'], 'bookTitle' => [ 'type' => 'text'], 'series' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'publicationTitle' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], - 'language' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], + 'language' => [ 'type' => 'keyword' ], 'place' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'date' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'archiveLocation' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], @@ -31,12 +31,7 @@ public static function getMappingParams(string $index): array 'type' => 'nested', 'properties' => [ 'creatorType' => [ - 'type' => 'text', - 'fields' => [ - 'keyword' => [ - 'type' => 'keyword', 'ignore_above' => 256 - ] - ] + 'type' => 'keyword' ], 'firstName' => [ 'type' => 'text', diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index 208f40f..e79a435 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -38,7 +38,7 @@ plugin.tx_lisztcommon_searchlisting { filters { 0 { field = itemType - type = terms + type = keyword } 1 { field = creators @@ -46,7 +46,7 @@ plugin.tx_lisztcommon_searchlisting { script ( String firstName = doc['creators.firstName.keyword'].size() > 0 ? doc['creators.firstName.keyword'].value : ''; String lastName = doc['creators.lastName.keyword'].size() > 0 ? doc['creators.lastName.keyword'].value : ''; - String type = doc['creators.creatorType.keyword'].size() > 0 ? doc['creators.creatorType.keyword'].value : ''; + String type = doc['creators.creatorType'].size() > 0 ? doc['creators.creatorType'].value : ''; if (type == 'translator') { return null; } @@ -62,7 +62,7 @@ plugin.tx_lisztcommon_searchlisting { } 3 { field = language - type = terms + type = keyword } 4 { field = publicationTitle From 34bdebed1b6093d2e950bb94cfac8b93fde3ac84 Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Mon, 6 Jan 2025 13:25:10 +0100 Subject: [PATCH 16/29] Change Mapping for separate FullName Fields --- Classes/Processing/BibElasticMapping.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Classes/Processing/BibElasticMapping.php b/Classes/Processing/BibElasticMapping.php index c911930..b470f77 100644 --- a/Classes/Processing/BibElasticMapping.php +++ b/Classes/Processing/BibElasticMapping.php @@ -15,18 +15,18 @@ public static function getMappingParams(string $index): array 'mappings' => [ 'dynamic' => false, 'properties' => [ - 'itemType' => [ 'type' => 'keyword' ], + 'itemType' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'version' => [ 'type' => 'long' ], 'title' => [ 'type' => 'text'], 'university' => [ 'type' => 'text'], 'bookTitle' => [ 'type' => 'text'], 'series' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'publicationTitle' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], - 'language' => [ 'type' => 'keyword' ], + 'language' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'place' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'date' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'archiveLocation' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], - 'journalTitle' => [ 'type' => 'keyword'], + 'journalTitle' => [ 'type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword', 'ignore_above' => 256 ] ] ], 'creators' => [ 'type' => 'nested', 'properties' => [ @@ -40,7 +40,6 @@ public static function getMappingParams(string $index): array 'type' => 'keyword', 'ignore_above' => 256 ], ], - 'copy_to' => 'creators.fullName' ], 'lastName' => [ 'type' => 'text', @@ -49,7 +48,6 @@ public static function getMappingParams(string $index): array 'type' => 'keyword', 'ignore_above' => 256 ] ], - 'copy_to' => 'creators.fullName' ], 'fullName' => ['type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword'] ] ], ] @@ -60,6 +58,13 @@ public static function getMappingParams(string $index): array 'tx_lisztcommon_footer' => [ 'type' => 'text' ], 'tx_lisztcommon_searchable' => ['type' => 'text', 'copy_to' => 'fulltext'], 'tx_lisztcommon_boosted' => ['type' => 'text'], + 'tx_lisztcommon_creators' => [ + 'type' => 'nested', + 'properties' => [ + 'fullName' => ['type' => 'text'], + ] + ], + ] ] ] From 2d07dc46edbd7f115b13703e5842ae4ec71217ef Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Mon, 6 Jan 2025 14:47:03 +0100 Subject: [PATCH 17/29] Add creators field while indexing --- Classes/Processing/BibElasticMapping.php | 13 +++++----- Classes/Processing/BibEntryConfig.php | 23 ++++++++++++++++++ Classes/Processing/BibEntryProcessor.php | 30 ++++++++++++++++++++---- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/Classes/Processing/BibElasticMapping.php b/Classes/Processing/BibElasticMapping.php index b470f77..99e124c 100644 --- a/Classes/Processing/BibElasticMapping.php +++ b/Classes/Processing/BibElasticMapping.php @@ -53,18 +53,17 @@ public static function getMappingParams(string $index): array ] ], 'fulltext' => [ 'type' => 'text' ], - 'tx_lisztcommon_header' => [ 'type' => 'text' ], - 'tx_lisztcommon_body' => [ 'type' => 'text' ], - 'tx_lisztcommon_footer' => [ 'type' => 'text' ], - 'tx_lisztcommon_searchable' => ['type' => 'text', 'copy_to' => 'fulltext'], - 'tx_lisztcommon_boosted' => ['type' => 'text'], - 'tx_lisztcommon_creators' => [ + BibEntryProcessor::HEADER_FIELD => [ 'type' => 'text' ], + BibEntryProcessor::BODY_FIELD => [ 'type' => 'text' ], + BibEntryProcessor::FOOTER_FIELD => [ 'type' => 'text' ], + BibEntryProcessor::SEARCHABLE_FIELD => ['type' => 'text', 'copy_to' => 'fulltext'], + BibEntryProcessor::BOOSTED_FIELD => ['type' => 'text'], + BibEntryProcessor::CREATORS_FIELD => [ 'type' => 'nested', 'properties' => [ 'fullName' => ['type' => 'text'], ] ], - ] ] ] diff --git a/Classes/Processing/BibEntryConfig.php b/Classes/Processing/BibEntryConfig.php index 108d7f4..1018209 100644 --- a/Classes/Processing/BibEntryConfig.php +++ b/Classes/Processing/BibEntryConfig.php @@ -216,6 +216,29 @@ class BibEntryConfig [ 'field' => 'date' ] ]; + const CREATORS_FIELD = [ + [ + 'compoundArray' => [ + 'fields' => [ + [ + 'field' => 'firstName', + 'conditionField' => 'creatorType', + 'conditionValue' => 'translator', + 'conditionRelation' => 'neq' + ], + [ + 'field' => 'lastName', + 'conditionField' => 'creatorType', + 'conditionValue' => 'translator', + 'conditionRelation' => 'neq' + ] + ], + 'field' => 'creators', + 'separator' => ' ' + ] + ] + ]; + public static function getAuthorHeader(): array { return [ self::AUTHOR ]; diff --git a/Classes/Processing/BibEntryProcessor.php b/Classes/Processing/BibEntryProcessor.php index 170d862..c11a9ec 100644 --- a/Classes/Processing/BibEntryProcessor.php +++ b/Classes/Processing/BibEntryProcessor.php @@ -20,6 +20,7 @@ class BibEntryProcessor extends IndexProcessor { + const CREATORS_FIELD = 'tx_lisztbibliography_creators'; public static function process( array $bibliographyItem, @@ -56,21 +57,27 @@ public static function process( $bibliographyItem[self::SEARCHABLE_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::SEARCHABLE_FIELDS); $bibliographyItem[self::BOOSTED_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::BOOSTED_FIELDS); + $bibliographyItem[self::CREATORS_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::CREATORS_FIELD); + return $bibliographyItem; } public static function buildListingField( array $bibliographyItem, array $fieldConfig - ): Stringable + ): Stringable|array { - return Collection::wrap($fieldConfig)-> - map( function($field) use ($bibliographyItem) { return self::buildListingEntry($field, $bibliographyItem); })-> + $collectedFields = Collection::wrap($fieldConfig)-> + map( function($field) use ($bibliographyItem) { return self::buildListingEntry($field, $bibliographyItem); }); + if (is_array($collectedFields->get(0))) { + return $collectedFields->get(0); + } + return $collectedFields-> join('')-> trim(); } - private static function buildListingEntry(array $field, array $bibliographyItem): ?Stringable + private static function buildListingEntry(array $field, array $bibliographyItem): Stringable|array|null { // return empty string if field does not exist if ( @@ -81,6 +88,21 @@ private static function buildListingEntry(array $field, array $bibliographyItem) ) { return null; } + + // return an array when compoundArray option is set + if (isset($field['compoundArray'])) { + // build compound fields + return Collection::wrap($bibliographyItem[$field['compoundArray']['field']])-> + // get selected strings + map(function ($bibliographyCell) use ($field) { + return self::processCompound($field['compoundArray'], $bibliographyCell); + })-> + // filter out non fitting fields + filter()-> + toArray(); + $bodyString = Str::of($compoundString); + } + // return empty string if conditions are not met if ( isset($field['conditionField']) && From b36764c23c70b3af0f396b24034d39c744166ce4 Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Mon, 6 Jan 2025 17:08:23 +0100 Subject: [PATCH 18/29] debug mapping and indexing (index only 3 docs) --- Classes/Processing/BibElasticMapping.php | 21 ++++++++++++++------- Configuration/TypoScript/setup.typoscript | 15 ++------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/Classes/Processing/BibElasticMapping.php b/Classes/Processing/BibElasticMapping.php index 99e124c..4c74328 100644 --- a/Classes/Processing/BibElasticMapping.php +++ b/Classes/Processing/BibElasticMapping.php @@ -53,15 +53,22 @@ public static function getMappingParams(string $index): array ] ], 'fulltext' => [ 'type' => 'text' ], - BibEntryProcessor::HEADER_FIELD => [ 'type' => 'text' ], - BibEntryProcessor::BODY_FIELD => [ 'type' => 'text' ], - BibEntryProcessor::FOOTER_FIELD => [ 'type' => 'text' ], - BibEntryProcessor::SEARCHABLE_FIELD => ['type' => 'text', 'copy_to' => 'fulltext'], - BibEntryProcessor::BOOSTED_FIELD => ['type' => 'text'], - BibEntryProcessor::CREATORS_FIELD => [ + 'tx_lisztcommon_header' => [ 'type' => 'text' ], + 'tx_lisztcommon_body' => [ 'type' => 'text' ], + 'tx_lisztcommon_footer' => [ 'type' => 'text' ], + 'tx_lisztcommon_searchable' => ['type' => 'text', 'copy_to' => 'fulltext'], + 'tx_lisztcommon_boosted' => ['type' => 'text'], + 'tx_lisztcommon_creators' => [ 'type' => 'nested', 'properties' => [ - 'fullName' => ['type' => 'text'], + 'fullName' => [ + 'type' => 'text', + 'fields' => [ + 'keyword' => [ + 'type' => 'keyword', 'ignore_above' => 256 + ], + ], + ], ] ], ] diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index e79a435..ed06157 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -41,20 +41,9 @@ plugin.tx_lisztcommon_searchlisting { type = keyword } 1 { - field = creators + field = tx_lisztcommon_creators type = nested - script ( - String firstName = doc['creators.firstName.keyword'].size() > 0 ? doc['creators.firstName.keyword'].value : ''; - String lastName = doc['creators.lastName.keyword'].size() > 0 ? doc['creators.lastName.keyword'].value : ''; - String type = doc['creators.creatorType'].size() > 0 ? doc['creators.creatorType'].value : ''; - if (type == 'translator') { - return null; - } - if (firstName == '' && lastName == '') { - return null; - } - return (firstName + ' ' + lastName).trim(); - ) + key = fullName } 2 { field = date From 4db61a30c162d914a70eb1bcb81665a27065e87a Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Wed, 8 Jan 2025 16:51:57 +0100 Subject: [PATCH 19/29] optimize mappping and handling of nested fields --- Classes/Processing/BibElasticMapping.php | 12 +++--- Classes/Processing/BibEntryProcessor.php | 50 ++++++++++++++++++++++- Configuration/TypoScript/setup.typoscript | 2 +- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/Classes/Processing/BibElasticMapping.php b/Classes/Processing/BibElasticMapping.php index 4c74328..faf2605 100644 --- a/Classes/Processing/BibElasticMapping.php +++ b/Classes/Processing/BibElasticMapping.php @@ -53,12 +53,12 @@ public static function getMappingParams(string $index): array ] ], 'fulltext' => [ 'type' => 'text' ], - 'tx_lisztcommon_header' => [ 'type' => 'text' ], - 'tx_lisztcommon_body' => [ 'type' => 'text' ], - 'tx_lisztcommon_footer' => [ 'type' => 'text' ], - 'tx_lisztcommon_searchable' => ['type' => 'text', 'copy_to' => 'fulltext'], - 'tx_lisztcommon_boosted' => ['type' => 'text'], - 'tx_lisztcommon_creators' => [ + BibEntryProcessor::HEADER_FIELD => [ 'type' => 'text' ], + BibEntryProcessor::BODY_FIELD => [ 'type' => 'text' ], + BibEntryProcessor::FOOTER_FIELD => [ 'type' => 'text' ], + BibEntryProcessor::SEARCHABLE_FIELD => ['type' => 'text', 'copy_to' => 'fulltext'], + BibEntryProcessor::BOOSTED_FIELD => ['type' => 'text'], + BibEntryProcessor::CREATORS_FIELD => [ 'type' => 'nested', 'properties' => [ 'fullName' => [ diff --git a/Classes/Processing/BibEntryProcessor.php b/Classes/Processing/BibEntryProcessor.php index c11a9ec..9d4c026 100644 --- a/Classes/Processing/BibEntryProcessor.php +++ b/Classes/Processing/BibEntryProcessor.php @@ -57,7 +57,7 @@ public static function process( $bibliographyItem[self::SEARCHABLE_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::SEARCHABLE_FIELDS); $bibliographyItem[self::BOOSTED_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::BOOSTED_FIELDS); - $bibliographyItem[self::CREATORS_FIELD] = self::buildListingField($bibliographyItem, BibEntryConfig::CREATORS_FIELD); + $bibliographyItem[self::CREATORS_FIELD] = self::buildNestedField($bibliographyItem, BibEntryConfig::CREATORS_FIELD); return $bibliographyItem; } @@ -70,6 +70,7 @@ public static function buildListingField( $collectedFields = Collection::wrap($fieldConfig)-> map( function($field) use ($bibliographyItem) { return self::buildListingEntry($field, $bibliographyItem); }); if (is_array($collectedFields->get(0))) { + print_r($collectedFields->get(0)); return $collectedFields->get(0); } return $collectedFields-> @@ -77,6 +78,53 @@ public static function buildListingField( trim(); } +/* + * @Matthias: this is a new function for nested fields with an array of object, + * ToDo: optimize and remove hard coded 'fullname' for tx_lisztbibliography_creators + * maybe ist much easier with an own BibEntryConfig +*/ + public static function buildNestedField( + array $bibliographyItem, + array $fieldConfig + ): Stringable|array + { + $collectedFields = Collection::wrap($fieldConfig) + ->map(function ($field) use ($bibliographyItem) { + return self::buildListingEntry($field, $bibliographyItem); + }); + + $result = $collectedFields->flatMap(function ($item) { + if (is_array($item)) { + return array_map(function ($i) { + // convert stringable to string if needed + if ($i instanceof Illuminate\Support\Stringable) { + $i = (string)$i; + } + return ['fullName' => $i]; + }, $item); + } + + // return fullName keys, ToDo: hardcoded key here + if ($item instanceof Illuminate\Support\Stringable) { + return [['fullName' => (string)$item]]; + } + + throw new \UnexpectedValueException('Unexpected type: ' . gettype($item)); + })->toArray(); + + return $result; + /*Array + returns: + ( + [0] => Array + ([fullName] => Illuminate\Support\Stringable Object ([value:protected] => Michael Saffle)) + + [1] => Array + ([fullName] => Illuminate\Support\Stringable Object([value:protected] => Michael Saffle) + ) + */ + } + private static function buildListingEntry(array $field, array $bibliographyItem): Stringable|array|null { // return empty string if field does not exist diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index ed06157..54f0f96 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -41,7 +41,7 @@ plugin.tx_lisztcommon_searchlisting { type = keyword } 1 { - field = tx_lisztcommon_creators + field = tx_lisztbibliography_creators type = nested key = fullName } From dbd35c63f9bc41f3d07358dac5d44852aee95266 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Mon, 6 Jan 2025 15:38:06 +0100 Subject: [PATCH 20/29] Fix --- Classes/Processing/BibEntryProcessor.php | 1 - 1 file changed, 1 deletion(-) diff --git a/Classes/Processing/BibEntryProcessor.php b/Classes/Processing/BibEntryProcessor.php index 9d4c026..422a926 100644 --- a/Classes/Processing/BibEntryProcessor.php +++ b/Classes/Processing/BibEntryProcessor.php @@ -148,7 +148,6 @@ private static function buildListingEntry(array $field, array $bibliographyItem) // filter out non fitting fields filter()-> toArray(); - $bodyString = Str::of($compoundString); } // return empty string if conditions are not met From 9c6394b0d1e5692ae6c1dc49b6771ebcbd56b86b Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Thu, 9 Jan 2025 17:18:57 +0100 Subject: [PATCH 21/29] Proposals for streamlining indexing --- Classes/Processing/BibElasticMapping.php | 4 +- Classes/Processing/BibEntryProcessor.php | 50 ++++++++++++------------ 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Classes/Processing/BibElasticMapping.php b/Classes/Processing/BibElasticMapping.php index faf2605..ea813e2 100644 --- a/Classes/Processing/BibElasticMapping.php +++ b/Classes/Processing/BibElasticMapping.php @@ -49,7 +49,7 @@ public static function getMappingParams(string $index): array ] ], ], - 'fullName' => ['type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword'] ] ], + BibEntryProcessor::FULLNAME_KEY => ['type' => 'text', 'fields' => [ 'keyword' => [ 'type' => 'keyword'] ] ], ] ], 'fulltext' => [ 'type' => 'text' ], @@ -61,7 +61,7 @@ public static function getMappingParams(string $index): array BibEntryProcessor::CREATORS_FIELD => [ 'type' => 'nested', 'properties' => [ - 'fullName' => [ + BibEntryProcessor::FULLNAME_KEY => [ 'type' => 'text', 'fields' => [ 'keyword' => [ diff --git a/Classes/Processing/BibEntryProcessor.php b/Classes/Processing/BibEntryProcessor.php index 422a926..0f1c899 100644 --- a/Classes/Processing/BibEntryProcessor.php +++ b/Classes/Processing/BibEntryProcessor.php @@ -21,6 +21,7 @@ class BibEntryProcessor extends IndexProcessor { const CREATORS_FIELD = 'tx_lisztbibliography_creators'; + const FULLNAME_KEY = 'fullName'; public static function process( array $bibliographyItem, @@ -65,7 +66,7 @@ public static function process( public static function buildListingField( array $bibliographyItem, array $fieldConfig - ): Stringable|array + ): Stringable { $collectedFields = Collection::wrap($fieldConfig)-> map( function($field) use ($bibliographyItem) { return self::buildListingEntry($field, $bibliographyItem); }); @@ -86,33 +87,34 @@ public static function buildListingField( public static function buildNestedField( array $bibliographyItem, array $fieldConfig - ): Stringable|array + ): array { - $collectedFields = Collection::wrap($fieldConfig) - ->map(function ($field) use ($bibliographyItem) { + return Collection::wrap($fieldConfig)-> + map(function ($field) use ($bibliographyItem) { return self::buildListingEntry($field, $bibliographyItem); - }); - - $result = $collectedFields->flatMap(function ($item) { - if (is_array($item)) { - return array_map(function ($i) { - // convert stringable to string if needed - if ($i instanceof Illuminate\Support\Stringable) { - $i = (string)$i; - } - return ['fullName' => $i]; - }, $item); - } - - // return fullName keys, ToDo: hardcoded key here - if ($item instanceof Illuminate\Support\Stringable) { - return [['fullName' => (string)$item]]; - } + })-> + flatMap(function (array $item): Collection { + //if (is_array($item)) { -> @Thomas do we need this? + return Collection::wrap($item)->map( function ($i) { + // convert stringable to string if needed -> @Thomas strings are autoconverted for me? + //if ($i instanceof Illuminate\Support\Stringable) { + //$i = (string)$i; + //} + return [self::FULLNAME_KEY => $i]; + }); + //} + + // return fullName keys, ToDo: hardcoded key here -> @Thomas do we need this? Can a stringable be supplied here? + if ($item instanceof Illuminate\Support\Stringable) { + return [[self::FULLNAME_KEY => (string)$item]]; + } + // @Thomas if we don't need the type checks, we can just return [self::FULLNAME_KEY => self::buildListingEntry($field, $bibliographyItem)] in the very first mapping function + // For me, indexing works without those type checks - throw new \UnexpectedValueException('Unexpected type: ' . gettype($item)); - })->toArray(); + throw new \UnexpectedValueException('Unexpected type: ' . gettype($item)); + })->toArray(); - return $result; + //return $result; /*Array returns: ( From 32b2a0643c42be2aa6c336a368b9b0bcb0b2cf3d Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Mon, 13 Jan 2025 10:46:02 +0100 Subject: [PATCH 22/29] new entity type 'multiselect' for filters in setup.typoscript Filters with this entity are multi selectable --- Configuration/TypoScript/setup.typoscript | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index 54f0f96..bce7923 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -34,28 +34,34 @@ plugin.tx_lisztcommon_searchlisting { extensionName = liszt_bibliography # the name of the entity index indexName = zotero + # the facetts are multi selectable if multiselect = yes # the filter fields filters { 0 { field = itemType type = keyword + multiselect = yes } 1 { field = tx_lisztbibliography_creators type = nested key = fullName + multiselect = yes } 2 { field = date type = terms + multiselect = yes } 3 { field = language type = keyword + multiselect = yes } 4 { field = publicationTitle type = terms + multiselect = yes } } } From 4dcc5beb2cc91522a874f60b0dd9e0519ace2e49 Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Tue, 14 Jan 2025 08:29:59 +0100 Subject: [PATCH 23/29] disable fetch localizedCitations and teiDataSets (temporary for more indexing speed) --- Classes/Command/IndexCommand.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php index 3a9ab55..37b95fd 100644 --- a/Classes/Command/IndexCommand.php +++ b/Classes/Command/IndexCommand.php @@ -239,8 +239,8 @@ protected function versionedSync(int $version): void protected function sync(int $cursor = 0, int $version = 0): void { $this->fetchBibliography($cursor, $version); - $this->fetchCitations($cursor, $version); - $this->fetchTeiData($cursor, $version); + // $this->fetchCitations($cursor, $version); + // $this->fetchTeiData($cursor, $version); $this->buildDataSets(); $this->commitBibliography(); } @@ -361,7 +361,13 @@ protected function buildDataSets(): void { $this->dataSets = $this->bibliographyItems-> map(function($bibliographyItem) { - return BibEntryProcessor::process($bibliographyItem, $this->localizedCitations, $this->teiDataSets); + return BibEntryProcessor::process( + $bibliographyItem, + new Collection(), + new Collection() + // $this->localizedCitations, + // $this->teiDataSets + ); }); } From d0222a508d106824a98163141a75618e851f90dd Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Tue, 14 Jan 2025 08:47:08 +0100 Subject: [PATCH 24/29] small refactoring for entityType multiselect (present or not, the value does not matter) --- Configuration/TypoScript/setup.typoscript | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index bce7923..af73d1f 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -34,34 +34,34 @@ plugin.tx_lisztcommon_searchlisting { extensionName = liszt_bibliography # the name of the entity index indexName = zotero - # the facetts are multi selectable if multiselect = yes + # the facets are multi selectable if multiselect is set # the filter fields filters { 0 { field = itemType type = keyword - multiselect = yes + multiselect= 1 } 1 { field = tx_lisztbibliography_creators type = nested key = fullName - multiselect = yes + multiselect= 1 } 2 { field = date type = terms - multiselect = yes + multiselect = 1 } 3 { field = language type = keyword - multiselect = yes + multiselect = 1 } 4 { field = publicationTitle type = terms - multiselect = yes + multiselect = 1 } } } From 25866011a8aab9bfc7c0542ad889ad22f60cd9c5 Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Tue, 14 Jan 2025 19:10:40 +0100 Subject: [PATCH 25/29] handle translations for bibliography facet items with own language fields --- Configuration/TypoScript/setup.typoscript | 2 ++ .../Private/Language/bibliographylang.xlf | 35 +++++++++++++++++++ .../Private/Language/de.bibliographylang.xlf | 11 ++++++ 3 files changed, 48 insertions(+) create mode 100644 Resources/Private/Language/bibliographylang.xlf create mode 100644 Resources/Private/Language/de.bibliographylang.xlf diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index af73d1f..8f4ce02 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -34,6 +34,8 @@ plugin.tx_lisztcommon_searchlisting { extensionName = liszt_bibliography # the name of the entity index indexName = zotero + # the filename of the language file for translations in /Resources/Private/Language/ of the extension + languageFile = bibliographylang # the facets are multi selectable if multiselect is set # the filter fields filters { diff --git a/Resources/Private/Language/bibliographylang.xlf b/Resources/Private/Language/bibliographylang.xlf new file mode 100644 index 0000000..be36c0c --- /dev/null +++ b/Resources/Private/Language/bibliographylang.xlf @@ -0,0 +1,35 @@ + + + +
+ + + Typ + + + Beteiligte + + + Datum + + + Sprache + + + Journaltitel + + + Buch + + + Journal + + + Buchbereich + + + Abschlussarbeit + + + + diff --git a/Resources/Private/Language/de.bibliographylang.xlf b/Resources/Private/Language/de.bibliographylang.xlf new file mode 100644 index 0000000..6d4ce06 --- /dev/null +++ b/Resources/Private/Language/de.bibliographylang.xlf @@ -0,0 +1,11 @@ + + + +
+ + + Kategorie + + + + From 21c52c22697533b0b24e3761df409a0425d5ef1c Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Wed, 15 Jan 2025 10:31:25 +0100 Subject: [PATCH 26/29] Fix Bug if no creators exists --- Classes/Processing/BibEntryProcessor.php | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Classes/Processing/BibEntryProcessor.php b/Classes/Processing/BibEntryProcessor.php index 0f1c899..1d02369 100644 --- a/Classes/Processing/BibEntryProcessor.php +++ b/Classes/Processing/BibEntryProcessor.php @@ -91,7 +91,21 @@ public static function buildNestedField( { return Collection::wrap($fieldConfig)-> map(function ($field) use ($bibliographyItem) { - return self::buildListingEntry($field, $bibliographyItem); + $entry = self::buildListingEntry($field, $bibliographyItem); + + // buildListingEntry can return null if no field creators exist + if ($entry === null) { + // ignore or return empty array ? + return []; // + } + + if (!is_array($entry)) { + throw new \UnexpectedValueException( + 'Expected array from buildListingEntry, but got: ' . gettype($entry) + ); + } + + return $entry; })-> flatMap(function (array $item): Collection { //if (is_array($item)) { -> @Thomas do we need this? @@ -134,7 +148,9 @@ private static function buildListingEntry(array $field, array $bibliographyItem) isset($field['field']) && !isset($bibliographyItem[$field['field']]) || isset($field['compound']['field']) && - !isset($bibliographyItem[$field['compound']['field']]) + !isset($bibliographyItem[$field['compound']['field']]) || + isset($field['compoundArray']['field']) && + !isset($bibliographyItem[$field['compoundArray']['field']]) ) { return null; } From d7965015ec4ac1faf7c2b819c1d4f95315bd087e Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Wed, 15 Jan 2025 10:35:14 +0100 Subject: [PATCH 27/29] clean up code --- Classes/Processing/BibEntryProcessor.php | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/Classes/Processing/BibEntryProcessor.php b/Classes/Processing/BibEntryProcessor.php index 1d02369..180097a 100644 --- a/Classes/Processing/BibEntryProcessor.php +++ b/Classes/Processing/BibEntryProcessor.php @@ -81,7 +81,6 @@ public static function buildListingField( /* * @Matthias: this is a new function for nested fields with an array of object, - * ToDo: optimize and remove hard coded 'fullname' for tx_lisztbibliography_creators * maybe ist much easier with an own BibEntryConfig */ public static function buildNestedField( @@ -104,32 +103,15 @@ public static function buildNestedField( 'Expected array from buildListingEntry, but got: ' . gettype($entry) ); } - return $entry; })-> flatMap(function (array $item): Collection { - //if (is_array($item)) { -> @Thomas do we need this? return Collection::wrap($item)->map( function ($i) { - // convert stringable to string if needed -> @Thomas strings are autoconverted for me? - //if ($i instanceof Illuminate\Support\Stringable) { - //$i = (string)$i; - //} return [self::FULLNAME_KEY => $i]; }); - //} - - // return fullName keys, ToDo: hardcoded key here -> @Thomas do we need this? Can a stringable be supplied here? - if ($item instanceof Illuminate\Support\Stringable) { - return [[self::FULLNAME_KEY => (string)$item]]; - } - // @Thomas if we don't need the type checks, we can just return [self::FULLNAME_KEY => self::buildListingEntry($field, $bibliographyItem)] in the very first mapping function - // For me, indexing works without those type checks - - throw new \UnexpectedValueException('Unexpected type: ' . gettype($item)); })->toArray(); - //return $result; - /*Array + /* returns: ( [0] => Array From 94041f49dbe11d0cb5b9eab25e525027753a474b Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Wed, 15 Jan 2025 10:51:29 +0100 Subject: [PATCH 28/29] update translations for itemTypes --- Resources/Private/Language/bibliographylang.xlf | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Resources/Private/Language/bibliographylang.xlf b/Resources/Private/Language/bibliographylang.xlf index be36c0c..1b97c7c 100644 --- a/Resources/Private/Language/bibliographylang.xlf +++ b/Resources/Private/Language/bibliographylang.xlf @@ -22,13 +22,22 @@ Buch - Journal + Zeitschriftenartikel - Buchbereich + Buchkapitel - Abschlussarbeit + Dissertation + + + Anhang + + + Webseite + + + Enzyklopädieartikel From c4995f028bf8f34e09ef1ed29e3926d362270171 Mon Sep 17 00:00:00 2001 From: thomas-sc Date: Wed, 15 Jan 2025 11:04:45 +0100 Subject: [PATCH 29/29] add entityType 'size' for number of facet items to display --- Configuration/TypoScript/setup.typoscript | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript index 8f4ce02..f74b6ae 100644 --- a/Configuration/TypoScript/setup.typoscript +++ b/Configuration/TypoScript/setup.typoscript @@ -37,6 +37,7 @@ plugin.tx_lisztcommon_searchlisting { # the filename of the language file for translations in /Resources/Private/Language/ of the extension languageFile = bibliographylang # the facets are multi selectable if multiselect is set + # size is the number of facet items, default = 10 # the filter fields filters { 0 { @@ -49,11 +50,13 @@ plugin.tx_lisztcommon_searchlisting { type = nested key = fullName multiselect= 1 + size = 15 } 2 { field = date type = terms multiselect = 1 + size = 15 } 3 { field = language