From 34d78c5396de795d7e4ad1e977c01f6b823840c7 Mon Sep 17 00:00:00 2001 From: rldhont Date: Wed, 4 Dec 2024 16:53:21 +0100 Subject: [PATCH] Speed up main view with project main data Adding a class to get only project main data for main view --- .../classes/listProjectDatasource.class.php | 2 +- .../lizmap/classes/lizmapRepository.class.php | 5 + .../lizmap/lib/Project/ProjectMainData.php | 562 ++++++++++++++++++ .../modules/lizmap/lib/Project/Repository.php | 56 ++ lizmap/modules/lizmap/lib/Server/Server.php | 2 +- lizmap/modules/view/zones/ajax_view.zone.php | 2 +- lizmap/modules/view/zones/main_view.zone.php | 2 +- .../classes/Project/ProjectMainDataTest.php | 81 +++ 8 files changed, 708 insertions(+), 4 deletions(-) create mode 100644 lizmap/modules/lizmap/lib/Project/ProjectMainData.php create mode 100644 tests/units/classes/Project/ProjectMainDataTest.php diff --git a/lizmap/modules/admin/classes/listProjectDatasource.class.php b/lizmap/modules/admin/classes/listProjectDatasource.class.php index 8a9addb308..7e8898a06b 100644 --- a/lizmap/modules/admin/classes/listProjectDatasource.class.php +++ b/lizmap/modules/admin/classes/listProjectDatasource.class.php @@ -26,7 +26,7 @@ public function getData($form) if ($criteria && array_key_exists($criteria, $this->data)) { $rep = lizmap::getRepository($criteria); // Get projects metadata - $metadata = $rep->getProjectsMetadata(); + $metadata = $rep->getProjectsMainData(); foreach ($metadata as $meta) { if ($meta->getHidden()) { continue; diff --git a/lizmap/modules/lizmap/classes/lizmapRepository.class.php b/lizmap/modules/lizmap/classes/lizmapRepository.class.php index c3546aa8ce..04c1df1f3e 100644 --- a/lizmap/modules/lizmap/classes/lizmapRepository.class.php +++ b/lizmap/modules/lizmap/classes/lizmapRepository.class.php @@ -166,6 +166,11 @@ public function getProjectsMetadata() return $this->repo->getProjectsMetadata(); } + public function getProjectsMainData() + { + return $this->repo->getProjectsMainData(); + } + /** * Return the value of the Access-Control-Allow-Origin HTTP header. * diff --git a/lizmap/modules/lizmap/lib/Project/ProjectMainData.php b/lizmap/modules/lizmap/lib/Project/ProjectMainData.php new file mode 100644 index 0000000000..edec65a9cd --- /dev/null +++ b/lizmap/modules/lizmap/lib/Project/ProjectMainData.php @@ -0,0 +1,562 @@ + $repository, + 'id' => $id, + 'title' => ucfirst($id), + 'abstract' => '', + 'keywordList' => '', + ); + + // Get project cached data + $qgsMtime = filemtime($file); + $qgsCfgMtime = filemtime($file.'.cfg'); + $cacheHandler = new ProjectCache($file, $qgsMtime, $qgsCfgMtime, $appContext); + $cachedData = $cacheHandler->retrieveProjectData(); + + if ($cachedData === false) { + // No project cached data read main data from files + $this->data = array_merge( + $data, + $this->readXmlProject($file), + $this->readCfgProject($file, $requiredTargetLwcVersion, $appContext), + ); + } else { + // Read main data from cached data + $this->data = array_merge( + $data, + $this->readCachedData($cachedData, $requiredTargetLwcVersion, $appContext), + ); + } + + if ($this->data['title'] == '') { + $this->data['title'] = ucfirst($id); + } + + $this->data['wmsGetCapabilitiesUrl'] = $appContext->getFullUrl( + 'lizmap~service:index', + array( + 'repository' => $repository, + 'project' => $id, + 'SERVICE' => 'WMS', + 'VERSION' => '1.3.0', + 'REQUEST' => 'GetCapabilities', + ) + ); + $this->data['wmtsGetCapabilitiesUrl'] = $appContext->getFullUrl( + 'lizmap~service:index', + array( + 'repository' => $repository, + 'project' => $id, + 'SERVICE' => 'WMTS', + 'VERSION' => '1.0.0', + 'REQUEST' => 'GetCapabilities', + ) + ); + + // Check right on repository + if ($this->data['acl'] && !$appContext->aclCheck('lizmap.repositories.view', $repository)) { + $this->data['acl'] = false; + } + } + + /** + * Get the project id. + * + * @return string the project id + */ + public function getId() + { + return $this->data['id']; + } + + /** + * Get the project repository code. + * + * @return string the project repository code + */ + public function getRepository() + { + return $this->data['repository']; + } + + /** + * Get the project title. + * + * @return string the project title + */ + public function getTitle() + { + return $this->data['title']; + } + + /** + * Get the project abstract. + * + * @return string the project abstract + */ + public function getAbstract() + { + return $this->data['abstract']; + } + + /** + * List of keywords. + * + * @return array + */ + public function getKeywordList() + { + return $this->data['keywordList']; + } + + /** + * FIXME what is the returned content ? + * + * @return mixed + */ + public function getProj() + { + return $this->data['proj']; + } + + /** + * Get the bounding box. + * + * FIXME what is the returned content ? + * + * @return mixed + */ + public function getBbox() + { + return $this->data['bbox']; + } + + /** + * Check if the project needs an update which is critical. + * + * @return bool true if the project needs an update + */ + public function needsUpdateError() + { + return $this->data['needsUpdateError']; + } + + /** + * Get the project access rights for the authenticated or anonymous user. + * + * @return bool True if the user has the right to access the Lizmap project + */ + public function getAcl() + { + return $this->data['acl']; + } + + /** + * Get the project hidden flag. + * + * @return bool True if the project must be hidden in the landing page + */ + public function getHidden() + { + return $this->data['hidden']; + } + + /** + * The url of WMS GetCapabilities. + * + * @return string + */ + public function getWMSGetCapabilitiesUrl() + { + return $this->data['wmsGetCapabilitiesUrl']; + } + + /** + * The url of WMTS GetCapabilities. + * + * @return string + */ + public function getWMTSGetCapabilitiesUrl() + { + return $this->data['wmtsGetCapabilitiesUrl']; + } + + /** + * The main data. + * + * @return array + */ + public function getData() + { + return array_merge(array(), $this->data); + } + + /** + * Read main data from QGIS file. + * + * @param string $qgs_path - the QGIS Project path + * + * @return array + */ + protected function readXmlProject($qgs_path) + { + $oXml = new \XMLReader(); + + // Open file + if (!$oXml->open($qgs_path)) { + throw new UnknownLizmapProjectException('The file '.$qgs_path.' cannot be parsed.'); + } + + $data = array(); + // Read until we are at the root document element + while ($oXml->read()) { + if ($oXml->nodeType == \XMLReader::ELEMENT + && $oXml->depth == 1 + && $oXml->localName == 'properties') { + $data = array_merge($data, $this->readXmlProperties($oXml)); + } + } + $errorMsg = ''; + foreach (libxml_get_errors() as $error) { + if ($errorMsg !== '') { + $errorMsg .= '\n'; + } + + switch ($error->level) { + case LIBXML_ERR_WARNING: + $errorMsg .= 'Warning '.$error->code.': '; + + break; + + case LIBXML_ERR_ERROR: + $errorMsg .= 'Error '.$error->code.': '; + + break; + + case LIBXML_ERR_FATAL: + $errorMsg .= 'Fatal Error '.$error->code.': '; + + break; + } + $errorMsg .= 'Line: '.$error->line.' '; + $errorMsg .= 'Column: '.$error->column.' '; + $errorMsg .= trim($error->message); + } + // Clear libxml error buffer + libxml_clear_errors(); + + if ($errorMsg !== '') { + throw new \Exception($errorMsg); + } + + return $data; + } + + /** + * Parse element. + * + * @param \XMLReader $oXmlReader An XMLReader instance at element + * + * @return array the result of the parsing + */ + protected function readXmlProperties($oXmlReader) + { + $depth = $oXmlReader->depth; + $localName = $oXmlReader->localName; + if ($oXmlReader->isEmptyElement) { + return array(); + } + + $data = array(); + while ($oXmlReader->read()) { + if ($oXmlReader->nodeType == \XMLReader::END_ELEMENT + && $oXmlReader->localName == $localName + && $oXmlReader->depth == $depth) { + break; + } + + if ($oXmlReader->nodeType != \XMLReader::ELEMENT) { + continue; + } + + if ($oXmlReader->depth != $depth + 1) { + continue; + } + + if ($oXmlReader->localName == 'WMSServiceTitle') { + $data['title'] = $oXmlReader->readString(); + } elseif ($oXmlReader->localName == 'WMSServiceAbstract') { + $data['abstract'] = $oXmlReader->readString(); + } elseif ($oXmlReader->localName == 'WMSKeywordList') { + $type = $oXmlReader->getAttribute('type'); + if ($type == 'QStringList') { + if (!$oXmlReader->isEmptyElement) { + $data['keywordList'] = implode(', ', $this->readValues($oXmlReader)); + } + } + } + } + + return $data; + } + + /** + * Parse elements. + * + * @param \XMLReader $oXmlReader An XMLReader instance at parent elements + * + * @return array the result of the parsing + */ + protected function readValues($oXmlReader) + { + if ($oXmlReader->nodeType != \XMLReader::ELEMENT) { + throw new \Exception('Provide an XMLReader::ELEMENT!'); + } + + $localName = $oXmlReader->localName; + $depth = $oXmlReader->depth; + $values = array(); + while ($oXmlReader->read()) { + if ($oXmlReader->nodeType == \XMLReader::END_ELEMENT + && $oXmlReader->localName == $localName + && $oXmlReader->depth == $depth) { + break; + } + + if ($oXmlReader->nodeType != \XMLReader::ELEMENT) { + continue; + } + + if ($oXmlReader->depth != $depth + 1) { + continue; + } + + if ($oXmlReader->localName == 'value') { + $values[] = $oXmlReader->readString(); + } + } + + return $values; + } + + /** + * Read main data from lizmap config file. + * + * @param string $qgs_path - the QGIS Project path + * @param int $requiredTargetLwcVersion - the configured required target Lizmap Web Client version + * @param App\AppContextInterface $appContext - the application context + * + * @return array + */ + protected function readCfgProject($qgs_path, $requiredTargetLwcVersion, $appContext) + { + $fileContent = file_get_contents($qgs_path.'.cfg'); + $cfgContent = json_decode($fileContent); + if ($cfgContent === null) { + throw new UnknownLizmapProjectException('The file '.$qgs_path.'.cfg cannot be decoded.'); + } + + return array( + 'proj' => $this->getProjOption($cfgContent), + 'bbox' => $this->getBboxOption($cfgContent), + 'needsUpdateError' => $this->getNeedsUpdateError($cfgContent, $requiredTargetLwcVersion), + 'acl' => $this->checkAcl($cfgContent, $appContext), + 'hidden' => $this->getHiddenOption($cfgContent), + ); + } + + /** + * Get option projection reference in a Lizmap config content. + * + * @param object $cfgContent - the Lizmap config content + * + * @return string + */ + protected function getProjOption($cfgContent) + { + if (property_exists($cfgContent, 'options') + && property_exists($cfgContent->options, 'projection') + && property_exists($cfgContent->options->projection, 'ref')) { + return $cfgContent->options->projection->ref; + } + + return ''; + } + + /** + * Get option bbox in a Lizmap config content. + * + * @param object $cfgContent - the Lizmap config content + * + * @return string + */ + protected function getBboxOption($cfgContent) + { + if (property_exists($cfgContent, 'options') + && property_exists($cfgContent->options, 'bbox')) { + return implode(', ', $cfgContent->options->bbox); + } + + return ''; + } + + /** + * Check if the project needs an update which lead to an error. + * + * @param object $cfgContent - the Lizmap config content + * @param int $requiredTargetLwcVersion - the configured required target Lizmap Web Client version + * + * @return bool true if the project needs to be updated in the QGIS desktop plugin + */ + protected function getNeedsUpdateError($cfgContent, $requiredTargetLwcVersion) + { + $lizmapWebClientTargetVersion = 30200; + if (property_exists($cfgContent, 'metadata') + && property_exists($cfgContent->metadata, 'lizmap_web_client_target_version')) { + $lizmapWebClientTargetVersion = $cfgContent->metadata->lizmap_web_client_target_version; + } + + return $lizmapWebClientTargetVersion < $requiredTargetLwcVersion; + } + + /** + * Check acl rights on the project. + * + * @param object $cfgContent - the Lizmap config content + * @param App\AppContextInterface $appContext - the application context + * + * @return bool true if the current user as rights on the project + */ + protected function checkAcl($cfgContent, $appContext) + { + // Check acl option is configured in project config + $aclGroups = null; + if (property_exists($cfgContent, 'options') + && property_exists($cfgContent->options, 'acl')) { + $aclGroups = $cfgContent->options->acl; + } + if ($aclGroups === null || !is_array($aclGroups) || empty($aclGroups)) { + return true; + } + + // Check user is authenticated + if (!$appContext->userIsConnected()) { + return false; + } + + // Check if configured groups white list and authenticated user groups list intersects + $userGroups = $appContext->aclUserGroupsId(); + if (array_intersect($aclGroups, $userGroups)) { + return true; + } + + return false; + } + + /** + * Get option hide project in a Lizmap config content. + * + * @param object $cfgContent - the Lizmap config content + * + * @return bool + */ + protected function getHiddenOption($cfgContent) + { + $value = ''; + if (property_exists($cfgContent, 'options') + && property_exists($cfgContent->options, 'hideProject')) { + $value = $cfgContent->options->hideProject; + } + if (empty($value)) { + return false; + } + if (is_bool($value)) { + return $value; + } + if (is_numeric($value)) { + return intval($value) > 0; + } + $strVal = strtolower($value); + + return in_array($strVal, array('true', 't', 'on', '1')); + } + + /** + * Read project main data from project cached data. + * + * @param array $cachedData - the Lizmap project cached data + * @param int $requiredTargetLwcVersion - the configured required target Lizmap Web Client version + * @param App\AppContextInterface $appContext - the application context + * + * @return array + */ + protected function readCachedData($cachedData, $requiredTargetLwcVersion, $appContext) + { + $cfgContent = $cachedData['cfg']; + + $data = array( + 'proj' => $this->getProjOption($cfgContent), + 'bbox' => $this->getBboxOption($cfgContent), + 'needsUpdateError' => $this->getNeedsUpdateError($cfgContent, $requiredTargetLwcVersion), + 'acl' => $this->checkAcl($cfgContent, $appContext), + 'hidden' => $this->getHiddenOption($cfgContent), + ); + + $qgisCachedData = $cachedData['qgis']['data']; + if (array_key_exists('title', $qgisCachedData)) { + $data['title'] = $qgisCachedData['title']; + } + if (array_key_exists('abstract', $qgisCachedData)) { + $data['abstract'] = $qgisCachedData['abstract']; + } + if (array_key_exists('keywordList', $qgisCachedData)) { + $data['keywordList'] = $qgisCachedData['keywordList']; + } + + return $data; + } +} diff --git a/lizmap/modules/lizmap/lib/Project/Repository.php b/lizmap/modules/lizmap/lib/Project/Repository.php index 9dcc890f12..96abb3371a 100644 --- a/lizmap/modules/lizmap/lib/Project/Repository.php +++ b/lizmap/modules/lizmap/lib/Project/Repository.php @@ -403,4 +403,60 @@ public function getProjectsMetadata() return $data; } + + /** + * Get the repository projects main data. + * + * @return ProjectMainData[] + */ + public function getProjectsMainData() + { + $data = array(); + $dir = $this->getPath(); + + if (is_dir($dir)) { + if ($dh = opendir($dir)) { + $cfgFiles = array(); + $qgsFiles = array(); + while (($file = readdir($dh)) !== false) { + if (substr($file, -3) == 'cfg') { + $cfgFiles[] = $file; + } + if (substr($file, -3) == 'qgs') { + $qgsFiles[] = $file; + } + } + closedir($dh); + + $requiredTargetLwcVersion = \jApp::config()->minimumRequiredVersion['lizmapWebClientTargetVersion']; + foreach ($qgsFiles as $qgsFile) { + $proj = null; + if (in_array($qgsFile.'.cfg', $cfgFiles)) { + try { + // Get project main data + $proj = new ProjectMainData( + $this->getKey(), + substr($qgsFile, 0, -4), + $this->getPath().$qgsFile, + $requiredTargetLwcVersion, + $this->appContext + ); + // $this->getProject(substr($qgsFile, 0, -4), $keepReference); + // Get the project metadata and add it to the returned object + // only if the authenticated user can access the project + if ($proj != null && $proj->getAcl()) { + $data[] = $proj; + } + } catch (UnknownLizmapProjectException $e) { + $this->appContext->logException($e, 'error'); + } catch (\Exception $e) { + $this->appContext->logException($e, 'error'); + } + } + } + } + } + + return $data; + } } diff --git a/lizmap/modules/lizmap/lib/Server/Server.php b/lizmap/modules/lizmap/lib/Server/Server.php index 12786ee373..abc37111ed 100644 --- a/lizmap/modules/lizmap/lib/Server/Server.php +++ b/lizmap/modules/lizmap/lib/Server/Server.php @@ -197,7 +197,7 @@ private function getLizmapRepositories() $repositories[$repositoryKey]['editing_authorized_groups'] = $editingAuthorizedGroups; // Add the projects - $repositoryProjects = $lizmapRepository->getProjectsMetadata(); + $repositoryProjects = $lizmapRepository->getProjectsMainData(); $projects = array(); foreach ($repositoryProjects as $project) { if (!$project->getAcl()) { diff --git a/lizmap/modules/view/zones/ajax_view.zone.php b/lizmap/modules/view/zones/ajax_view.zone.php index e77378c360..adceba83ab 100644 --- a/lizmap/modules/view/zones/ajax_view.zone.php +++ b/lizmap/modules/view/zones/ajax_view.zone.php @@ -40,7 +40,7 @@ protected function _prepareTpl() $lrep = lizmap::getRepository($r); $mrep = new lizmapMainViewItem($r, $lrep->getLabel()); - $metadata = $lrep->getProjectsMetadata(); + $metadata = $lrep->getProjectsMainData(); foreach ($metadata as $meta) { // Avoid project with no access rights if (!$meta->getAcl()) { diff --git a/lizmap/modules/view/zones/main_view.zone.php b/lizmap/modules/view/zones/main_view.zone.php index 5aac378298..9f5864c976 100644 --- a/lizmap/modules/view/zones/main_view.zone.php +++ b/lizmap/modules/view/zones/main_view.zone.php @@ -61,7 +61,7 @@ protected function _prepareTpl() // Get all files name in the repository directory to look for thumbnails $repFiles = scandir($lrep->getPath()); - $metadata = $lrep->getProjectsMetadata(); + $metadata = $lrep->getProjectsMainData(); foreach ($metadata as $meta) { // Avoid project which needs an update if ($meta->needsUpdateError()) { diff --git a/tests/units/classes/Project/ProjectMainDataTest.php b/tests/units/classes/Project/ProjectMainDataTest.php new file mode 100644 index 0000000000..76adbd0e7d --- /dev/null +++ b/tests/units/classes/Project/ProjectMainDataTest.php @@ -0,0 +1,81 @@ +assertEquals('tests', $p->getRepository()); + $this->assertEquals('montpellier', $p->getId()); + $this->assertEquals('Montpellier - Transports', $p->getTitle()); + $this->assertStringStartsWith('Demo project with bus and tramway lines in Montpellier', $p->getAbstract()); + $this->assertEquals('', $p->getKeywordList()); + $this->assertEquals('USER:100000', $p->getProj()); + $this->assertEquals('417006.6137376, 5394910.340903, 447158.04891101, 5414844.9948054', $p->getBbox()); + $this->assertFalse($p->needsUpdateError()); + $this->assertTrue($p->getAcl()); + $this->assertFalse($p->getHidden()); + //$this->assertEquals(array(), $p->getData()); + + $file = __DIR__.'/Ressources/montpellier_intranet.qgs'; + $p = new Project\ProjectMainData('tests', 'montpellier_intranet', $file, 30200, $context); + $this->assertEquals('tests', $p->getRepository()); + $this->assertEquals('montpellier_intranet', $p->getId()); + $this->assertEquals('Montpellier - Intranet map example', $p->getTitle()); + $this->assertStringStartsWith('Some data from OpenDataMontpellier shown on a map', $p->getAbstract()); + $this->assertEquals('', $p->getKeywordList()); + $this->assertEquals('EPSG:4326', $p->getProj()); + $this->assertEquals('3.78300108, 43.54854151, 3.97065725, 43.67333749', $p->getBbox()); + $this->assertFalse($p->needsUpdateError()); + $this->assertTrue($p->getAcl()); + $this->assertFalse($p->getHidden()); + + $file = __DIR__.'/Ressources/events.qgs'; + $p = new Project\ProjectMainData('tests', 'events', $file, 30800, $context); + $this->assertEquals('tests', $p->getRepository()); + $this->assertEquals('events', $p->getId()); + $this->assertEquals('Touristic events around Montpellier, France', $p->getTitle()); + $this->assertEquals('', $p->getAbstract()); + $this->assertEquals('', $p->getKeywordList()); + $this->assertEquals('EPSG:4242', $p->getProj()); + $this->assertEquals('390483.99668047, 5375009.91444, 477899.47320636, 5436768.5630521', $p->getBbox()); + $this->assertTrue($p->needsUpdateError()); + $this->assertTrue($p->getAcl()); + $this->assertFalse($p->getHidden()); + + $file = __DIR__.'/Ressources/embed_parent.qgs'; + $p = new Project\ProjectMainData('tests', 'embed_parent', $file, 30600, $context); + $this->assertEquals('tests', $p->getRepository()); + $this->assertEquals('embed_parent', $p->getId()); + $this->assertEquals('embed_parent', $p->getTitle()); + $this->assertEquals('', $p->getAbstract()); + $this->assertEquals('', $p->getKeywordList()); + $this->assertEquals('EPSG:4326', $p->getProj()); + $this->assertEquals('3.72495035999999979, 43.54176487699999853, 4.03651796000000029, 43.68444261700000197', $p->getBbox()); + $this->assertFalse($p->needsUpdateError()); + $this->assertTrue($p->getAcl()); + $this->assertFalse($p->getHidden()); + + $file = __DIR__.'/../../../qgis-projects/tests/test_tags_nature_flower.qgs'; + $p = new Project\ProjectMainData('tests', 'test_tags_nature_flower', $file, 30600, $context); + $this->assertEquals('tests', $p->getRepository()); + $this->assertEquals('test_tags_nature_flower', $p->getId()); + $this->assertEquals('Test tags: nature, flower', $p->getTitle()); + $this->assertEquals('This is an abstract', $p->getAbstract()); + $this->assertEquals('nature, flower', $p->getKeywordList()); + $this->assertEquals('EPSG:4326', $p->getProj()); + $this->assertEquals('-1.2459627329192546, -1.0, 1.2459627329192546, 1.0', $p->getBbox()); + $this->assertFalse($p->needsUpdateError()); + $this->assertTrue($p->getAcl()); + $this->assertFalse($p->getHidden()); + } +}