Skip to content

Commit

Permalink
Index localised citations and TEI data
Browse files Browse the repository at this point in the history
  • Loading branch information
Matthias Richter committed Apr 24, 2024
1 parent 0b34ab7 commit 1708843
Showing 1 changed file with 153 additions and 59 deletions.
212 changes: 153 additions & 59 deletions Classes/Command/IndexCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,48 @@
use Elasticsearch\Client;
use Hedii\ZoteroApi\ZoteroApi;
use Illuminate\Support\Collection;
use Psr\Http\Message\ServerRequestInterface;
use Slub\LisztCommon\Common\ElasticClientBuilder;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
use TYPO3\CMS\Core\Localization\Locale;
use TYPO3\CMS\Core\Site\Entity\Site;
use TYPO3\CMS\Core\Site\Entity\SiteLanguage;
use TYPO3\CMS\Core\Site\SiteFinder;
use TYPO3\CMS\Core\Utility\GeneralUtility;

class IndexCommand extends Command
{

protected ZoteroApi $bibApi;
protected string $apiKey;
protected Collection $bibliographyItems;
protected Collection $teiDataSets;
protected Collection $dataSets;
protected Client $client;
protected array $extConf;
protected SymfonyStyle $io;
protected ZoteroApi $localeApi;
protected array $locales;
protected int $bulkSize;
protected int $total;
protected Collection $locales;
protected Collection $localizedCitations;

function __construct(SiteFinder $siteFinder)
{
parent::__construct();

$this->locales = Collection::wrap($siteFinder->getAllSites())->
map(function (Site $site): array { return $site->getLanguages(); })->
flatten()->
map(function (SiteLanguage $language): string { return $language->getHreflang(); });
}

protected function getRequest(): ServerRequestInterface
{
return $GLOBALS['TYPO3_REQUEST'];
}

protected function configure(): void
{
Expand All @@ -45,35 +67,76 @@ protected function configure(): void
protected function initialize(InputInterface $input, OutputInterface $output) {
$this->extConf = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('liszt_bibliography');
$this->client = ElasticClientBuilder::getClient();
$this->bibApi = new ZoteroApi($this->extConf['zoteroApiKey']);
$this->localeApi = new ZoteroApi($this->extConf['zoteroApiKey']);
$this->apiKey = $this->extConf['zoteroApiKey'];
$this->io = new SymfonyStyle($input, $output);
$this->io->title($this->getDescription());
}

protected function execute(InputInterface $input, OutputInterface $output): int
{
// get bulk size and total size
$this->bulkSize = (int) $this->extConf['zoteroBulkSize'];

$this->io->section('Fetching Bibliography Data');
$this->fetchBibliography();
$this->io->section('Complementing export formats');
$this->complementExportFormats();
$this->io->section('Fetching Localized Citations');
$this->fetchCitations();
$this->io->section('Fetching TEI Data');
$this->fetchTeiData();
$this->io->section('Building Datasets');
$this->buildDataSets();
$this->io->section('Committing Bibliography Data');
$this->commitBibliography();
$this->io->section('Committing Locale Data');
$this->commitLocales();
return 0;
}

protected function complementExportFormats(): void
protected function buildDataSets(): void
{
$this->io->progressStart($this->total);
$this->dataSets = $this->bibliographyItems->
map(function($bibliographyItem) {
$this->io->progressAdvance();
return self::buildDataSet($bibliographyItem, $this->localizedCitations, $this->teiDataSets);
});
$this->io->progressFinish();
}

protected static function buildDataSet(
array $bibliographyItem,
Collection $localizedCitations,
Collection $teiDataSets
)
{
$key = $bibliographyItem['key'];
$bibliographyItem['localizedCitations'] = [];
foreach ($localizedCitations as $locale => $localizedCitation) {
$bibliographyItem['localizedCitations'][$locale] = $localizedCitation->get($key)['citation'];
}
$bibliographyItem['tei'] = $teiDataSets->get($key);
return $bibliographyItem;
}

protected function fetchBibliography(): void
{
$client = new ZoteroApi($this->extConf['zoteroApiKey']);
$response = $client->
group($this->extConf['zoteroGroupId'])->
items()->
top()->
limit(1)->
send();
$this->total = (int) $response->getHeaders()['Total-Results'][0];

// fetch bibliography items bulkwise
$this->io->progressStart($this->total);
$collection = new Collection($response->getBody());
$this->bibliographyItems = $collection->pluck('data');
$this->bibliographyItems = $collection->
pluck('data');

$cursor = $this->bulkSize;
while ($cursor < $this->total) {
$this->io->progressAdvance($this->bulkSize);
$response = $this->bibApi->
$response = $client->
group($this->extConf['zoteroGroupId'])->
items()->
top()->
Expand All @@ -88,43 +151,99 @@ protected function complementExportFormats(): void
$this->io->progressFinish();
}

protected function fetchBibliography(): void
protected function fetchCitations(): void
{
// fetch locales
$response = $this->localeApi->
raw('https://api.zotero.org/schema?format=json')->
$this->localizedCitations = new Collection();
$this->locales->each(function($locale) { $this->fetchCitationLocale($locale); });
}

protected function fetchCitationLocale(string $locale): void
{
$client = new ZoteroApi($this->extConf['zoteroApiKey']);
$style = $this->extConf['zoteroStyle'];
$response = $client->
group($this->extConf['zoteroGroupId'])->
items()->
top()->
limit(1)->
setInclude('citation')->
setStyle($style)->
setLinkwrap()->
setLocale($locale)->
send();
$this->locales = $response->getBody()['locales'];

// get bulk size and total size
$this->bulkSize = (int) $this->extConf['zoteroBulkSize'];
$response = $this->bibApi->
// fetch bibliography items bulkwise
$this->io->text($locale);
$this->io->progressStart($this->total);
$result = Collection::wrap($response->getBody())->keyBy('key');

$cursor = $this->bulkSize;
while ($cursor < $this->total) {
try {
$response = $client->
group($this->extConf['zoteroGroupId'])->
items()->
top()->
start($cursor)->
limit($this->bulkSize)->
setInclude('citation')->
setStyle($style)->
setLinkwrap()->
setLocale($locale)->
send();
$result = $result->merge(Collection::wrap($response->getBody())->keyBy('key'));
$cursor += $this->bulkSize;
} catch (\Exception $e) {
$this->io->newline(2);
$this->io->caution($e->getMessage());
$this->io->note('Stay calm. This is normal for Zotero\'s API. I\'m trying it again.');
}
$this->io->progressAdvance($this->bulkSize);
}

$this->localizedCitations = $this->localizedCitations->merge(
new Collection([ $locale => $result ])
);
$this->io->progressFinish();
}

protected function fetchTeiData(): void
{
$client = new ZoteroApi($this->extConf['zoteroApiKey']);
$response = $client->
group($this->extConf['zoteroGroupId'])->
items()->
top()->
limit(1)->
setInclude('tei')->
send();
$this->total = (int) $response->getHeaders()['Total-Results'][0];

// fetch bibliography items bulkwise
$this->io->progressStart($this->total);
$collection = new Collection($response->getBody());
$this->bibliographyItems = $collection->pluck('data');
$this->teiDataSets = $collection->keyBy('key');

$cursor = $this->bulkSize;
while ($cursor < $this->total) {
try {
$response = $client->
group($this->extConf['zoteroGroupId'])->
items()->
top()->
start($cursor)->
limit($this->bulkSize)->
setInclude('tei')->
send();
$collection = new Collection($response->getBody());
$this->teiDataSets = $this->teiDataSets->
concat($collection->keyBy('key'));
$cursor += $this->bulkSize;
} catch (\Exception $e) {
$this->io->newline(2);
$this->io->caution($e->getMessage());
$this->io->note('Stay calm. This is normal for Zotero\'s API. I\'m trying it again.');
}
$this->io->progressAdvance($this->bulkSize);
$response = $this->bibApi->
group($this->extConf['zoteroGroupId'])->
items()->
top()->
start($cursor)->
limit($this->bulkSize)->
send();
$collection = new Collection($response->getBody());
$this->bibliographyItems = $this->bibliographyItems->
concat($collection->pluck('data'));
$cursor += $this->bulkSize;
}
$this->io->progressFinish();
}
Expand All @@ -142,7 +261,7 @@ protected function commitBibliography(): void

$params = [ 'body' => [] ];
$bulkCount = 0;
foreach ($this->bibliographyItems as $document) {
foreach ($this->dataSets as $document) {
$this->io->progressAdvance();
$params['body'][] = [ 'index' =>
[
Expand All @@ -163,29 +282,4 @@ protected function commitBibliography(): void
$this->io->text('done');
}

protected function commitLocales(): void
{
$localeIndex = $this->extConf['elasticLocaleIndexName'];
$this->io->text('Committing the ' . $localeIndex . ' index');

if ($this->client->indices()->exists(['index' => $localeIndex])) {
$this->client->indices()->delete(['index' => $localeIndex]);
$this->client->indices()->create(['index' => $localeIndex]);
}

$params = [ 'body' => [] ];
foreach ($this->locales as $key => $locale) {
$params['body'][] = [ 'index' =>
[
'_index' => $localeIndex,
'_id' => $key
]
];
$params['body'][] = json_encode($locale);

}
$this->client->bulk($params);

$this->io->text('done');
}
}

0 comments on commit 1708843

Please sign in to comment.