diff --git a/Classes/Command/IndexCommand.php b/Classes/Command/IndexCommand.php new file mode 100644 index 0000000..e8be191 --- /dev/null +++ b/Classes/Command/IndexCommand.php @@ -0,0 +1,163 @@ +setDescription('Create elasticsearch index from zotero bibliography'); + } + + protected function initialize(InputInterface $input, OutputInterface $output) { + $this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography'); + $this->client = ElasticClientBuilder::getClient(); + $this->bibApi = new ZoteroApi($this->extConf['zoteroApiKey']); + $this->localeApi = new ZoteroApi($this->extConf['zoteroApiKey']); + $this->io = new SymfonyStyle($input, $output); + $this->io->title($this->getDescription()); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $this->io->section('Fetching Bibliography Data'); + $this->fetchBibliography(); + $this->io->section('Committing Bibliography Data'); + $this->commitBibliography(); + $this->io->section('Committing Locale Data'); + $this->commitLocales(); + return 0; + } + + protected function fetchBibliography(): void + { + // fetch locales + $response = $this->localeApi-> + raw('https://api.zotero.org/schema?format=json')-> + send(); + $this->locales = $response->getBody()['locales']; + + // get bulk size and total size + $bulkSize = (int) $this->extConf['zoteroBulkSize']; + $response = $this->bibApi-> + group($this->extConf['zoteroGroupId'])-> + items()-> + top()-> + limit(1)-> + send(); + $total = (int) $response->getHeaders()['Total-Results'][0]; + + // fetch bibliography items bulkwise + $this->io->progressStart($total); + $collection = new Collection($response->getBody()); + $this->bibliographyItems = $collection->pluck('data'); + + $cursor = $bulkSize; + while ($cursor < $total) { + $this->io->progressAdvance($bulkSize); + $response = $this->bibApi-> + group($this->extConf['zoteroGroupId'])-> + items()-> + top()-> + start($cursor)-> + limit($bulkSize)-> + send(); + $collection = new Collection($response->getBody()); + $this->bibliographyItems = $this->bibliographyItems-> + concat($collection->pluck('data')); + $cursor += $bulkSize; + } + $this->io->progressFinish(); + } + + protected function commitBibliography(): void + { + $index = $this->extConf['elasticIndexName']; + $this->io->text('Committing the ' . $index . ' index'); + + $this->io->progressStart(count($this->bibliographyItems)); + if ($this->client->indices()->exists(['index' => $index])) { + $this->client->indices()->delete(['index' => $index]); + $this->client->indices()->create(['index' => $index]); + } + + $params = [ 'body' => [] ]; + $bulkCount = 0; + foreach ($this->bibliographyItems as $document) { + $this->io->progressAdvance(); + $params['body'][] = [ 'index' => + [ + '_index' => $index, + '_id' => $document['key'] + ] + ]; + $params['body'][] = json_encode($document); + + if (!(++$bulkCount % $this->extConf['elasticBulkSize'])) { + $this->client->bulk($params); + $params = [ 'body' => [] ]; + } + } + $this->io->progressFinish(); + $this->client->bulk($params); + + $this->io->text('done'); + } + + protected function commitLocales(): void + { + $localeIndex = $this->extConf['elasticLocaleIndexName']; + $this->io->text('Committing the ' . $localeIndex . ' index'); + + if ($this->client->indices()->exists(['index' => $localeIndex])) { + $this->client->indices()->delete(['index' => $localeIndex]); + $this->client->indices()->create(['index' => $localeIndex]); + } + + $params = [ 'body' => [] ]; + foreach ($this->locales as $key => $locale) { + $params['body'][] = [ 'index' => + [ + '_index' => $localeIndex, + '_id' => $key + ] + ]; + $params['body'][] = json_encode($locale); + + } + $this->client->bulk($params); + + $this->io->text('done'); + } +} diff --git a/Classes/Controller/BibliographyController.php b/Classes/Controller/BibliographyController.php new file mode 100644 index 0000000..2e0511a --- /dev/null +++ b/Classes/Controller/BibliographyController.php @@ -0,0 +1,93 @@ +responseFactory = $responseFactory; + $this->streamFactory = $streamFactory; + } + + public function initializeAction(): void + { + $extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography'); + $this->bibIndex = $extConf['elasticIndexName']; + $this->localeIndex = $extConf['elasticLocaleIndexName']; + } + + public function indexAction(): ResponseInterface + { + $this->createJsCall(); + $this->wrapTargetDiv(); + $contentStream = $this-> + streamFactory-> + createStream( + $this->div . + $this->jsCall + ); + + return $this-> + responseFactory-> + createResponse()-> + withBody($contentStream); + } + + private function wrapTargetDiv(): void + { + $sideCol = '
'; + $mainCol = '
'; + $this->div = '
' . + $sideCol . $mainCol . '
'; + } + + private function createJsCall(): void + { + $this->jsCall = + ''; + } +} diff --git a/Configuration/Services.yaml b/Configuration/Services.yaml new file mode 100644 index 0000000..b621b14 --- /dev/null +++ b/Configuration/Services.yaml @@ -0,0 +1,18 @@ +services: + _defaults: + autowire: true + autoconfigure: true + public: false + Slub\LisztBibliography\: + resource: '../Classes/*' + exclude: '../Classes/Domain/Model/*' + Slub\LisztBibliography\Classes\Controller\BibliographyController: + tags: + - backend.controller + Slub\LisztBibliography\Command\IndexCommand: + tags: + - + name: console.command + command: 'liszt-bibliography:index' + description: 'Create elasticsearch index from zotero bibliography' + schedulable: true diff --git a/Configuration/TCA/Overrides/tt_content.php b/Configuration/TCA/Overrides/tt_content.php new file mode 100644 index 0000000..4c454d7 --- /dev/null +++ b/Configuration/TCA/Overrides/tt_content.php @@ -0,0 +1,45 @@ + ' + --div--;LLL:EXT:core/Resources/Private/Language/Form/locallang_tabs.xlf:general, + --palette--;;general, + header; Title, + bodytext;LLL:EXT:core/Resources/Private/Language/Form/locallang_ttc.xlf:bodytext_formlabel, + --div--;LLL:EXT:core/Resources/Private/Language/Form/locallang_tabs.xlf:access, + --palette--;;hidden, + --palette--;;acces, + ', + 'columnsOverrides' => [ + 'bodytext' => [ + 'config' => [ + 'enableRichtext' => true, + 'richtextConfiguration' => 'default' + ] + ] + ] +]; + diff --git a/Configuration/TsConfig/Page/Mod/Wizards/Listing.tsconfig b/Configuration/TsConfig/Page/Mod/Wizards/Listing.tsconfig new file mode 100644 index 0000000..1ea5e22 --- /dev/null +++ b/Configuration/TsConfig/Page/Mod/Wizards/Listing.tsconfig @@ -0,0 +1,28 @@ +mod.wizards.newContentElement.wizardItems { + plugins { + elements { + lisztbibliography_bibliographylisting { + iconIdentifier = content-text + title = LLL:EXT:liszt_bibliography/Resources/Private/Language/locallang.xlf:listing_title + description = LLL:EXT:liszt_bibliography/Resources/Private/Language/locallang.xlf:listing_description + tt_content_defValues { + CType = list + list_type = lisztbibliography_bibliographylisting + } + } + } + } + common { + elements { + lisztbibliography_bibliographylisting { + iconIdentifier = content-text + title = LLL:EXT:liszt_bibliography/Resources/Private/Language/locallang.xlf:listing_title + description = LLL:EXT:liszt_bibliography/Resources/Private/Language/locallang.xlf:listing_description + tt_content_defValues { + CType = lisztbibliography_listing + } + } + } + show := addToList(lisztbibliography_listing) + } +} diff --git a/Configuration/TypoScript/setup.typoscript b/Configuration/TypoScript/setup.typoscript new file mode 100644 index 0000000..d6de7b9 --- /dev/null +++ b/Configuration/TypoScript/setup.typoscript @@ -0,0 +1,25 @@ +###################### +#### DEPENDENCIES #### +###################### + + +lib.contentElement { + #layoutRootPaths { + #200 = EXT:liszt_bibliography:Resources/Private/Layouts + #} + #partialRootPaths { + #200 = EXT:liszt_bibliography:Resources/Private/Partial + #} + templateRootPaths { + 200 = EXT:liszt_bibliography/Resources/Private/Templates + } +} + +tt_content { + lisztbibliography_listing =< lib.contentElement + lisztbibliography_listing { + templateName = BibliographyListing + } +} + +page.includeJSFooter.BibliographyController = EXT:liszt_bibliography/Resources/Public/JavaScript/Src/BibliographyController.js diff --git a/Resources/Private/Language/locallang.xlf b/Resources/Private/Language/locallang.xlf new file mode 100644 index 0000000..ac8e907 --- /dev/null +++ b/Resources/Private/Language/locallang.xlf @@ -0,0 +1,14 @@ + + + +
+ + + Bibliographie-Listing + + + Eine Auflistung aller Einträge der Liszt-Bibliographie + + + + diff --git a/Resources/Public/JavaScript/Src/BibliographyController.js b/Resources/Public/JavaScript/Src/BibliographyController.js new file mode 100644 index 0000000..a21419f --- /dev/null +++ b/Resources/Public/JavaScript/Src/BibliographyController.js @@ -0,0 +1,155 @@ +/** todos + * fehlermeldungen + * ad hoc funktionen benennen + * facette entstehungsdatum + * t3-buildmechanismus für js + * facettenregistratur + * urlparameter auf highlighting abbilden + */ + +const LISZT_BIB_DEFAULT_SIZE = 20; +const LISZT_BIB_DEFAULT_TRIGGER_POSITION = 200; +const LISZT_BIB_DEFAULT_BIBLIST_ID = 'bib-list'; +const LISZT_BIB_DEFAULT_SIDELIST_ID = 'bib-list-side'; +const LISZT_BIB_DEFAULT_BIBINDEX = 'zotero'; +const LISZT_BIB_DEFAULT_LOCALEINDEX = 'zotero'; + +class BibliographyController { + + #urlManager = null; + #target = ''; + #size = LISZT_BIB_DEFAULT_SIZE; + #triggerPosition = LISZT_BIB_DEFAULT_TRIGGER_POSITION; + #bibListId = LISZT_BIB_DEFAULT_BIBLIST_ID; + #url = new URL(location); + #body = {}; + + constructor (config) { + this.#target = config.target; + this.#size = config.size ?? this.#size; + this.#triggerPosition = config.triggerPosition ?? this.#triggerPosition; + this.#bibListId = config.bibListId ?? this.#bibListId; + + this.init(); + } + + init() { + this.#body = {}; + this.#urlManager = new UrlManager(); + this.#urlManager.registerMapping({itemTypes: 'query.match.itemType'}); + this.#body = this.#urlManager.body; + + this.client = new elasticsearch.Client({ + host: 'https://ddev-liszt-portal.ddev.site:9201' + }); + this.from = 0; + this.docs = this.client.search({ index: 'zotero', size: this.#size, body: this.#body }); + this.docs.then(docs => this.renderDocs(docs, this.#target)); + + const types = this.client.search({ + index:'zotero', + size:0, + body:{ + aggs:{ + types:{ + categorize_text:{ + field:'itemType' + } + } + } + } + }); + types.then(r => this.renderTypes(r.aggregations.types.buckets)); + + let allowed = true; + $(window).scroll(_ => { + const bottomPosition = $(document).scrollTop() + $(window).height() + if (bottomPosition > $(document).height() - this.#triggerPosition && allowed) { + allowed = false; + this.from += this.#size; + const newDocs = this.client.search({index:'zotero',size:this.#size,from:this.from,body:this.#body}); + newDocs.then(docs => { + this.appendDocs(docs, this.#target); + allowed = true; + }); + } + }); + } + + renderTypes(buckets) { + const render = buckets => { + const renderedBuckets = buckets + .sort((a, b) => (b.doc_count - a.doc_count)) + .map(bucket => this.renderType(bucket, this.locales)) + .join(''); + const buttonGroup = `
${renderedBuckets}
`; + $(`#bib-list-side`).append(`

  • ${this.locales.fields.itemType}

    • `); + $(`#bib-list-side #item-type`).append(buttonGroup); + $(`#bib-list-side .list-group-item-action`).click(d => { + d.preventDefault(); + this.from = 0; + $(d.target).toggleClass('active'); + const matches = $(`#bib-list-side .list-group-item.active`).map((d,i) => $(i).attr('data')); + const itemTypes = matches.toArray().join(' '); + + this.#urlManager.setParam('itemTypes', itemTypes); + this.#body = this.#urlManager.body; + const docs = this.client.search({index:'zotero',size:this.#size,from:this.from,body:this.#body}); + docs.then(docs => { + this.renderDocs(docs, this.#target); + }); + }); + } + if (!this.locales) { + this.client.get({index:'zoterolocales',id:'de'}).then(locales => { + this.locales = locales._source; + render(buckets); + }); + } else { + render(buckets); + } + } + + renderType(bucket, locales) { + const key = locales.itemTypes[bucket.key]; + return ` ${key} (${bucket.doc_count}) `; + } + + appendDocs(docs, target) { + const hits = docs.hits.hits.map(hit => hit._source); + const renderedDocs = hits.map(hit => BibliographyController.renderDoc(hit, this.locales)).join(''); + $(renderedDocs).hide().appendTo(`#${target} #${this.#bibListId}`).fadeIn('slow'); + + } + + renderDocs(docs, target) { + const render = (docs, target) => { + const hits = docs.hits.hits.map(hit => hit._source); + const renderedDocs = hits.map(hit => BibliographyController.renderDoc(hit, this.locales)).join(''); + $(`#${target}`).html(`
      ${renderedDocs}
    `); + } + if (!this.locales) { + this.client.get({index:'zoterolocales',id:'de'}).then(locales => { + this.locales = locales._source; + render(docs, target) + }); + } else { + render(docs, target); + } + } + + static renderDoc(doc, locales) { + const itemType = locales.itemTypes[doc.itemType]; + const renderedCreators = doc.creators ? BibliographyController.renderCreators(doc.creators, locales) : ''; + return `

  • ${doc.title} ${itemType}

    ${renderedCreators}
    • `; + } + + static renderCreators(creators, locales) { + return creators.map(creator => BibliographyController.renderCreator(creator, locales)).join(', '); + } + + static renderCreator(creator, locales) { + const creatorType = locales.creatorTypes[creator.creatorType]; + return `${creator.firstName} ${creator.lastName} (${creatorType})`; + } +} diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..f2acc24 --- /dev/null +++ b/composer.json @@ -0,0 +1,25 @@ +{ + "name": "slub/liszt-bibliography", + "description": "Manages an elasticsearch index obtained from a zotero library and other data sources, displays a bibliography listing", + "type": "typo3-cms-extension", + "license": [ + "GPL-2.0-or-later" + ], + "require": { + "typo3/cms-core": "^11.5", + "typo3/cms-fluid-styled-content": "^11.5", + "slub/liszt-common": "dev-main", + "dikastes/zotero-api": "dev-master", + "illuminate/collections": "^8" + }, + "autoload": { + "psr-4": { + "Slub\\LisztBibliography\\": "Classes/" + } + }, + "extra": { + "typo3/cms": { + "extension-key": "liszt_bibliography" + } + } +} diff --git a/ext_conf_template.txt b/ext_conf_template.txt new file mode 100644 index 0000000..c6bb055 --- /dev/null +++ b/ext_conf_template.txt @@ -0,0 +1,14 @@ +# cat=Elasticsearch; type=string; label=LLL:EXT:publisher_db/Resources/Private/Language/Labels.xml:config.elasticIndexName +elasticIndexName = zotero +# cat=Elasticsearch; type=string; label=LLL:EXT:publisher_db/Resources/Private/Language/Labels.xml:config.elasticLocaleIndexName +elasticLocaleIndexName = zoterolocales +# cat=Elasticsearch; type=int; label=LLL:EXT:publisher_db/Resources/Private/Language/Labels.xml:config.elasticBulkSize +elasticBulkSize = 20 +# cat=Zotero; type=string; label=LLL:EXT:liszt_zotero/Resources/Private/Language/Labels.xml:config.zoteroApiKey +zoteroApiKey = xgC1nXqyHO2eYOq2M7Dj7V5q +# cat=Zotero; type=int; label=LLL:EXT:liszt_zotero/Resources/Private/Language/Labels.xml:config.zoteroGroupId +zoteroGroupId = 5080468 +# cat=Zotero; type=int; label=LLL:EXT:liszt_zotero/Resources/Private/Language/Labels.xml:config.zoteroUserId +zoteroUserId = 7071210 +# cat=Zotero; type=int; label=LLL:EXT:liszt_zotero/Resources/Private/Language/Labels.xml:config.zoteroBulkSize +zoteroBulkSize = 50 diff --git a/ext_localconf.php b/ext_localconf.php new file mode 100644 index 0000000..e0cc1d0 --- /dev/null +++ b/ext_localconf.php @@ -0,0 +1,20 @@ + 'index' ], + [ BibliographyController::class => 'index' ] +); + +ExtensionManagementUtility::addPageTSConfig( + '' +);