From 1077b60e99d0146465a635eea209b5be082d5ff5 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 1 Jan 2024 08:37:51 -0500 Subject: [PATCH 01/24] =?UTF-8?q?Happy=20New=20Year!=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LICENSE.txt | 2 +- docs/conf.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 1c4b83026..0008a6ef7 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,7 +1,7 @@ The MIT License (MIT) ===================== -Copyright © 2010-2023 Tom Kralidis +Copyright © 2010-2024 Tom Kralidis Copyright © 2011-2021 Angelos Tzotsos Copyright © 2012-2015 Adam Hinz Copyright © 2015-2021 Ricardo Garcia Silva diff --git a/docs/conf.py b/docs/conf.py index 2d7c4e5d0..b9a0e1c9f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,7 +3,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -81,7 +81,7 @@ project = u'pycsw' authors = u'Tom Kralidis' license = u'This work is licensed under a Creative Commons Attribution 4.0 International License' -copyright = u'2010-2023, ' + authors + ' ' + license +copyright = u'2010-2024, ' + authors + ' ' + license # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From cd8e49d15ba14ccb3917da72a79ad9c0d479c2a0 Mon Sep 17 00:00:00 2001 From: Vincent Fazio Date: Mon, 1 Jan 2024 09:27:00 +1100 Subject: [PATCH 02/24] Fix incorrect XML tag name in apiso --- pycsw/plugins/profiles/apiso/apiso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycsw/plugins/profiles/apiso/apiso.py b/pycsw/plugins/profiles/apiso/apiso.py index 6334cefd1..638923be6 100644 --- a/pycsw/plugins/profiles/apiso/apiso.py +++ b/pycsw/plugins/profiles/apiso/apiso.py @@ -112,7 +112,7 @@ def __init__(self, model, namespaces, context): 'apiso:Creator': {'xpath': 'gmd:identificationInfo/gmd:MD_DataIdentification/gmd:pointOfContact/gmd:CI_ResponsibleParty/gmd:organisationName[gmd:role/gmd:CI_RoleCode/@codeListValue="originator"]/gco:CharacterString', 'dbcol': self.context.md_core_model['mappings']['pycsw:Creator']}, 'apiso:Publisher': {'xpath': 'gmd:identificationInfo/gmd:MD_DataIdentification/gmd:pointOfContact/gmd:CI_ResponsibleParty/gmd:organisationName[gmd:role/gmd:CI_RoleCode/@codeListValue="publisher"]/gco:CharacterString', 'dbcol': self.context.md_core_model['mappings']['pycsw:Publisher']}, 'apiso:Contributor': {'xpath': 'gmd:identificationInfo/gmd:MD_DataIdentification/gmd:pointOfContact/gmd:CI_ResponsibleParty/gmd:organisationName[gmd:role/gmd:CI_RoleCode/@codeListValue="contributor"]/gco:CharacterString', 'dbcol': self.context.md_core_model['mappings']['pycsw:Contributor']}, - 'apiso:Relation': {'xpath': 'gmd:identificationInfo/gmd:MD_Data_Identification/gmd:aggregationInfo', 'dbcol': self.context.md_core_model['mappings']['pycsw:Relation']}, + 'apiso:Relation': {'xpath': 'gmd:identificationInfo/gmd:MD_DataIdentification/gmd:aggregationInfo', 'dbcol': self.context.md_core_model['mappings']['pycsw:Relation']}, # 19115-2 'apiso:Platform': {'xpath': 'gmi:acquisitionInfo/gmi:MI_AcquisitionInformation/gmi:platform/gmi:MI_Platform/gmi:identifier', 'dbcol': self.context.md_core_model['mappings']['pycsw:Platform']}, 'apiso:Instrument': {'xpath': 'gmi:acquisitionInfo/gmi:MI_AcquisitionInformation/gmi:platform/gmi:MI_Platform/gmi:instrument/gmi:MI_Instrument/gmi:identifier', 'dbcol': self.context.md_core_model['mappings']['pycsw:Instrument']}, From 2390abcd6a9438ddab260ce399584ed56fda2cda Mon Sep 17 00:00:00 2001 From: Vincent Fazio Date: Mon, 1 Jan 2024 09:39:24 +1100 Subject: [PATCH 03/24] Correct misleading 'setup-repository' command in docs --- docs/administration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.rst b/docs/administration.rst index 0daf3ccdd..8009cf13a 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -54,7 +54,7 @@ Setting up the Database .. code-block:: bash - pycsw-admin.py setup_repository --config default.cfg + pycsw-admin.py setup-repository --config default.cfg This will create the necessary tables and values for the repository. From 88d73ffd86c5a136715b0524ad0bd725f65aaec8 Mon Sep 17 00:00:00 2001 From: Vincent Fazio Date: Mon, 1 Jan 2024 09:14:43 +1100 Subject: [PATCH 04/24] Update python versions for tox.ini to be same as github actions --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a174a0c46..d6c5d3747 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = {py36,py37,py38,py39}-sqlite +envlist = {py38,py39,py310,py311}-sqlite skip_missing_interpreters = True [testenv] From 7e568cf546699b96fa00085d66ff7972d9098ced Mon Sep 17 00:00:00 2001 From: Barel Shmuely Date: Wed, 27 Dec 2023 14:31:31 +0200 Subject: [PATCH 05/24] fix parse of nested multiple binary ops (#782) --- pycsw/ogc/fes/fes1.py | 3 +- pycsw/ogc/fes/fes2.py | 3 +- ...tRecords-filter-and-nested-or-multiple.xml | 19 +++++++++++ ...tRecords-filter-and-nested-or-multiple.xml | 32 +++++++++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 tests/functionaltests/suites/default/expected/post_GetRecords-filter-and-nested-or-multiple.xml create mode 100644 tests/functionaltests/suites/default/post/GetRecords-filter-and-nested-or-multiple.xml diff --git a/pycsw/ogc/fes/fes1.py b/pycsw/ogc/fes/fes1.py index d6268aa8e..2e443d84b 100644 --- a/pycsw/ogc/fes/fes1.py +++ b/pycsw/ogc/fes/fes1.py @@ -236,7 +236,6 @@ def _get_comparison_expression(elem): return expression queries = [] - queries_nested = [] values = [] LOGGER.debug('Scanning children elements') @@ -301,8 +300,10 @@ def _get_comparison_expression(elem): tagname = ' %s ' % child_tag_name.lower() if tagname in [' or ', ' and ']: # this is a nested binary logic query LOGGER.debug('Nested binary logic detected; operator=%s', tagname) + queries_nested = [] for child2 in child.xpath('child::*'): queries_nested.append(_get_comparison_expression(child2)) + LOGGER.debug('Nested binary logic queries: %s', queries_nested) queries.append('(%s)' % tagname.join(queries_nested)) else: queries.append(_get_comparison_expression(child)) diff --git a/pycsw/ogc/fes/fes2.py b/pycsw/ogc/fes/fes2.py index c38abf4e1..a29dcb946 100644 --- a/pycsw/ogc/fes/fes2.py +++ b/pycsw/ogc/fes/fes2.py @@ -254,7 +254,6 @@ def _get_comparison_expression(elem): return expression queries = [] - queries_nested = [] values = [] LOGGER.debug('Scanning children elements') @@ -319,8 +318,10 @@ def _get_comparison_expression(elem): tagname = ' %s ' % child_tag_name.lower() if tagname in [' or ', ' and ']: # this is a nested binary logic query LOGGER.debug('Nested binary logic detected; operator=%s', tagname) + queries_nested = [] for child2 in child.xpath('child::*'): queries_nested.append(_get_comparison_expression(child2)) + LOGGER.debug('Nested binary logic queries: %s', queries_nested) queries.append('(%s)' % tagname.join(queries_nested)) else: queries.append(_get_comparison_expression(child)) diff --git a/tests/functionaltests/suites/default/expected/post_GetRecords-filter-and-nested-or-multiple.xml b/tests/functionaltests/suites/default/expected/post_GetRecords-filter-and-nested-or-multiple.xml new file mode 100644 index 000000000..42fbc5950 --- /dev/null +++ b/tests/functionaltests/suites/default/expected/post_GetRecords-filter-and-nested-or-multiple.xml @@ -0,0 +1,19 @@ + + + + + + + urn:uuid:94bc9c83-97f6-4b40-9eb8-a8e8787a5c63 + http://purl.org/dc/dcmitype/Dataset + Mauris sed neque + Vegetation-Cropland + Curabitur lacinia, ante non porta tempus, mi lorem feugiat odio, eget suscipit eros pede ac velit. + 2006-03-26 + + 47.595 -4.097 + 51.217 0.889 + + + + diff --git a/tests/functionaltests/suites/default/post/GetRecords-filter-and-nested-or-multiple.xml b/tests/functionaltests/suites/default/post/GetRecords-filter-and-nested-or-multiple.xml new file mode 100644 index 000000000..3ab5276b5 --- /dev/null +++ b/tests/functionaltests/suites/default/post/GetRecords-filter-and-nested-or-multiple.xml @@ -0,0 +1,32 @@ + + + + full + + + + + + dc:type + http://purl.org/dc/dcmitype/Image + + + dc:type + http://purl.org/dc/dcmitype/Dataset + + + + + dc:title + Mauris% + + + dc:title + %neque + + + + + + + From 1702f77bee339999efd111dc57fe11767e475336 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 6 Jan 2024 10:27:44 -0500 Subject: [PATCH 06/24] update configuration to YAML (#937) * update configuration to YAML (#797) * pass OARec tests * STAC updates * fix refs * pass CITE tests * pass APISO tests * pass Atom tests * pass APISO INSPIRE tests * pass idswithpaths tests * pass default tests * pass dif tests * pass duplicatefileid tests * pass ebrim tests * pass fgdc tests * pass gm03 tests * update harvesting tests * pass manager tests * pass oaipmh tests * pass opensearcheo tests * pass repofilter tests * pass sru tests * update utf-8 tests * updates * fix * Fixed failing tests * update docker configs * docs sweep * add migration script * add yaml dump to file * Fix typo in helm configmap * Update k8s configmap * Update default config file * Update config files * update copyright year --------- Co-authored-by: Ricardo Garcia Silva Co-authored-by: Angelos Tzotsos --- default-sample.cfg | 101 ----------- default-sample.yml | 125 +++++++++++++ docker/compose/docker-compose.yml | 2 +- docker/compose/pycsw.cfg | 102 ----------- docker/compose/pycsw.yml | 114 ++++++++++++ docker/entrypoint.py | 37 ++-- docker/helm/templates/pycsw-configmap.yaml | 138 ++++++++------- docker/helm/values.yaml | 87 +++++---- docker/kubernetes/pycsw-configmap.yaml | 163 +++++++++-------- docker/pycsw.cfg | 102 ----------- docker/pycsw.yml | 115 ++++++++++++ docs/administration.rst | 52 +++--- docs/api.rst | 7 +- docs/configuration.rst | 95 +++++----- docs/distributedsearching.rst | 37 ++-- docs/docker.rst | 10 +- docs/hhypermap.rst | 2 +- docs/installation.rst | 22 +-- docs/introduction.rst | 4 +- docs/json.rst | 6 +- docs/migration-guide.rst | 3 +- docs/oaipmh.rst | 4 +- docs/oarec-support.rst | 4 +- docs/opensearch.rst | 4 +- docs/profiles.rst | 2 +- docs/repofilters.rst | 14 +- docs/repositories.rst | 8 +- docs/sitemaps.rst | 2 +- docs/sru.rst | 4 +- docs/tools.rst | 6 +- docs/xslt.rst | 10 +- pycsw/core/admin.py | 137 +++++++++++++-- pycsw/core/log.py | 85 ++++----- pycsw/core/repository.py | 3 +- pycsw/oaipmh.py | 6 +- pycsw/ogc/api/oapi.py | 20 +-- pycsw/ogc/api/records.py | 67 ++++--- pycsw/ogc/api/templates/_base.html | 10 +- pycsw/ogc/api/templates/landing_page.html | 2 +- pycsw/ogc/api/templates/openapi.html | 2 +- pycsw/ogc/api/util.py | 26 ++- pycsw/ogc/csw/csw2.py | 102 +++++------ pycsw/ogc/csw/csw3.py | 95 +++++----- pycsw/opensearch.py | 22 +-- pycsw/plugins/profiles/apiso/apiso.py | 51 +++--- pycsw/plugins/profiles/apiso/docs/apiso.rst | 8 +- pycsw/plugins/profiles/ebrim/docs/ebrim.rst | 2 +- pycsw/plugins/profiles/ebrim/ebrim.py | 4 +- pycsw/plugins/profiles/profile.py | 4 +- pycsw/server.py | 157 +++++++---------- pycsw/stac/api.py | 17 +- pycsw/wsgi.py | 15 +- pycsw/wsgi_flask.py | 13 +- requirements.txt | 1 + tests/functionaltests/conftest.py | 31 ++-- .../suites/apiso-inspire/default.cfg | 91 ---------- .../suites/apiso-inspire/default.yml | 103 +++++++++++ .../expected/get_GetCapabilities-lang.xml | 24 +-- .../expected/get_GetCapabilities.xml | 24 +-- .../functionaltests/suites/apiso/default.cfg | 91 ---------- .../functionaltests/suites/apiso/default.yml | 103 +++++++++++ .../apiso/expected/post_GetCapabilities.xml | 22 +-- tests/functionaltests/suites/atom/default.cfg | 90 ---------- tests/functionaltests/suites/atom/default.yml | 100 +++++++++++ .../expected/get_opensearch-description.xml | 2 +- .../get_opensearch-ogc-bbox-and-time.xml | 6 +- .../atom/expected/get_opensearch-ogc-bbox.xml | 8 +- .../get_opensearch-ogc-count-and-page1.xml | 6 +- .../get_opensearch-ogc-count-and-page2.xml | 4 +- .../get_opensearch-ogc-q-and-bbox.xml | 4 +- .../get_opensearch-ogc-q-and-time.xml | 4 +- .../atom/expected/get_opensearch-ogc-q.xml | 4 +- .../atom/expected/get_opensearch-ogc-time.xml | 4 +- .../expected/get_opensearch-ogc-timeend.xml | 4 +- .../expected/get_opensearch-ogc-timestart.xml | 8 +- .../suites/atom/expected/get_opensearch.xml | 22 +-- .../atom/expected/post_GetCapabilities.xml | 22 +-- .../expected/post_GetRecords-filter-bbox.xml | 4 +- tests/functionaltests/suites/cite/default.cfg | 86 --------- tests/functionaltests/suites/cite/default.yml | 101 +++++++++++ ...t_27e17158-c57a-4493-92ac-dba8934cf462.xml | 26 +-- ...t_2ab7d1fa-885b-459f-80e4-b6282eab4f8c.xml | 26 +-- ...t_477b23a3-baa9-47c8-9541-5fe27735ed49.xml | 24 +-- ...t_48f26761-3a9d-48db-bee1-da089f5fb857.xml | 26 +-- ...t_55c38f00-2553-42c1-99ab-33edbb561ad7.xml | 24 +-- ...t_6c375703-9c00-4aef-bec7-d2e964f849eb.xml | 2 +- ...t_80f31def-4185-48b9-983a-960566918eae.xml | 24 +-- ...t_9697f0aa-3b6a-4125-83a5-61e8826127c4.xml | 24 +-- ...t_ba5fc729-3b71-47a0-b7d0-42ec565cd185.xml | 26 +-- ...t_f4692ec5-9547-4a05-88ab-e6154af2640a.xml | 26 +-- ...t_7c89cdf5-0def-4cfb-8c55-2b8ffea5d92f.xml | 10 +- .../functionaltests/suites/csw30/default.cfg | 90 ---------- .../functionaltests/suites/csw30/default.yml | 100 +++++++++++ ...t_002258f0-627f-457f-b2ad-025777c77ac8.xml | 4 +- ...t_045c600d-973d-41eb-9f60-eba1b717b720.xml | 6 +- ...t_0bbcf862-5211-4351-9988-63f8bec49c98.xml | 24 +-- ...t_0bdf8457-971e-4ed1-be4a-5feca4dcd8fa.xml | 20 +-- ...t_0e1dca37-477a-4060-99fe-7799b52d656c.xml | 4 +- ...t_22f44168-2ccf-4801-ad96-204212566d56.xml | 20 +-- ...t_2499a9c9-8d33-449c-bc92-d494adfcc84d.xml | 20 +-- ...t_27f4f39c-d92a-4e3c-b961-c6aa8c24e513.xml | 20 +-- ...t_2b06a5c8-0df2-4af1-8d2e-a425de11c845.xml | 20 +-- ...t_2ba1418a-444d-4cce-9cfe-4c94efcf8b55.xml | 8 +- ...t_43cd6471-6ac7-45bd-8ff9-148cb2de9a52.xml | 20 +-- ...t_5e9e67dc-18d6-4645-8111-c6263c88a61f.xml | 20 +-- ...t_6a9d0558-9d87-495b-b999-b49a3ef1cf99.xml | 20 +-- ...t_6e9cba43-5e27-415d-adbd-a92851c2c173.xml | 20 +-- ...t_7e82446a-b5dc-43fe-9a73-4cc1f2f2f0bf.xml | 20 +-- ...t_8025978e-1a35-4d70-80c2-e8329e0c7864.xml | 20 +-- ...t_8e5fa0f6-3f29-4d1f-abe2-d9866f3def98.xml | 10 +- .../expected/get_GetCapabilities-base-url.xml | 20 +-- .../get_GetCapabilities-no-version.xml | 20 +-- .../csw30/expected/get_GetCapabilities.xml | 20 +-- .../expected/get_OpenSearch-description.xml | 4 +- ...t_a2f18643-e24e-4fa5-b780-6de4a2dbc814.xml | 2 +- ...t_b2aafc3f-4f35-47bc-affd-08590972deae.xml | 10 +- ...t_b6069623-f7d8-4021-8582-98f0aea0f763.xml | 12 +- ...t_c03d173a-3f42-4956-89c8-1fe02c3a0873.xml | 20 +-- ...t_dc246fb8-5af5-4fda-82bb-c18b3ecd439c.xml | 10 +- ...t_de016645-6d5c-4855-943c-2db07ae9f49a.xml | 6 +- ...t_e67ca935-d65d-4d8c-8302-1405333dded0.xml | 20 +-- .../csw30/expected/post_GetCapabilities.xml | 20 +-- .../suites/csw30/get/requests.txt | 16 +- .../suites/default/default.cfg | 90 ---------- .../suites/default/default.yml | 100 +++++++++++ .../default/expected/get_GetCapabilities.xml | 22 +-- .../expected/post_GetCapabilities-SOAP.xml | 22 +-- .../post_GetCapabilities-updatesequence.xml | 22 +-- .../default/expected/post_GetCapabilities.xml | 22 +-- tests/functionaltests/suites/dif/default.cfg | 90 ---------- tests/functionaltests/suites/dif/default.yml | 100 +++++++++++ .../dif/expected/post_GetCapabilities.xml | 22 +-- .../suites/duplicatefileid/default.cfg | 90 ---------- .../suites/duplicatefileid/default.yml | 100 +++++++++++ .../expected/get_GetCapabilities.xml | 22 +-- .../functionaltests/suites/ebrim/default.cfg | 93 ---------- .../functionaltests/suites/ebrim/default.yml | 103 +++++++++++ .../ebrim/expected/post_GetCapabilities.xml | 22 +-- tests/functionaltests/suites/fgdc/default.cfg | 90 ---------- tests/functionaltests/suites/fgdc/default.yml | 100 +++++++++++ .../fgdc/expected/post_GetCapabilities.xml | 22 +-- tests/functionaltests/suites/gm03/default.cfg | 90 ---------- tests/functionaltests/suites/gm03/default.yml | 100 +++++++++++ .../gm03/expected/post_GetCapabilities.xml | 22 +-- .../suites/harvesting/default.cfg | 91 ---------- .../suites/harvesting/default.yml | 103 +++++++++++ .../expected/post_GetCapabilities.xml | 26 +-- ...est-zzz-post-GetRecords-filter-sos-iso.xml | 60 +++---- ...est-zzz-post-GetRecords-filter-wfs-iso.xml | 30 ++-- ...est-zzz-post-GetRecords-filter-wms-iso.xml | 6 +- .../suites/harvesting/get/requests.txt | 10 +- .../suites/idswithpaths/default.cfg | 90 ---------- .../suites/idswithpaths/default.yml | 100 +++++++++++ .../expected/get_GetCapabilities.xml | 22 +-- .../suites/manager/default.cfg | 91 ---------- .../suites/manager/default.yml | 103 +++++++++++ .../manager/expected/post_GetCapabilities.xml | 26 +-- .../functionaltests/suites/oaipmh/default.cfg | 91 ---------- .../functionaltests/suites/oaipmh/default.yml | 103 +++++++++++ .../get_GetRecord_bad_metadata_prefix.xml | 2 +- .../expected/get_GetRecord_datacite.xml | 2 +- .../oaipmh/expected/get_GetRecord_dc.xml | 2 +- .../oaipmh/expected/get_GetRecord_iso.xml | 2 +- .../oaipmh/expected/get_GetRecord_oai_dc.xml | 2 +- .../suites/oaipmh/expected/get_Identify.xml | 4 +- ...et_ListIdentifiers_bad_metadata_prefix.xml | 2 +- .../expected/get_ListIdentifiers_datacite.xml | 2 +- .../expected/get_ListIdentifiers_dc.xml | 2 +- .../expected/get_ListIdentifiers_iso.xml | 2 +- ...istIdentifiers_missing_metadata_prefix.xml | 2 +- .../expected/get_ListIdentifiers_oai_dc.xml | 2 +- .../expected/get_ListMetadataFormats.xml | 2 +- .../expected/get_ListRecords_datacite.xml | 2 +- .../oaipmh/expected/get_ListRecords_dc.xml | 2 +- ...get_ListRecords_dc_bad_metadata_prefix.xml | 2 +- .../expected/get_ListRecords_iso19139.xml | 2 +- .../expected/get_ListRecords_oai_dc.xml | 2 +- .../suites/oaipmh/expected/get_ListSets.xml | 2 +- .../suites/oaipmh/expected/get_bad_verb.xml | 2 +- .../suites/oaipmh/expected/get_empty.xml | 2 +- .../oaipmh/expected/get_empty_with_amp.xml | 2 +- .../oaipmh/expected/get_illegal_verb.xml | 2 +- .../functionaltests/suites/oarec/conftest.py | 165 ++++++++++++------ .../suites/opensearcheo/default.cfg | 91 ---------- .../suites/opensearcheo/default.yml | 103 +++++++++++ .../get_opensearch-description-document.xml | 4 +- .../get_opensearch-query-cloudcover-gt.xml | 4 +- .../get_opensearch-query-cloudcover-lt-gt.xml | 4 +- .../get_opensearch-query-cloudcover-lt.xml | 6 +- .../get_opensearch-query-cloudcover.xml | 6 +- .../get_opensearch-query-instrument.xml | 6 +- .../get_opensearch-query-orbitdirection.xml | 6 +- .../get_opensearch-query-orbitnumber.xml | 6 +- .../get_opensearch-query-platform.xml | 6 +- .../get_opensearch-query-processinglevel.xml | 6 +- .../get_opensearch-query-producttype.xml | 6 +- .../get_opensearch-query-sensortype.xml | 6 +- .../get_opensearch-query-snowcover.xml | 6 +- .../get_opensearch-query-spectralrange.xml | 6 +- ...get_opensearch-query-start-stop-extent.xml | 24 +-- .../get_opensearch-query-time-extent.xml | 24 +-- .../suites/repofilter/default.cfg | 91 ---------- .../suites/repofilter/default.yml | 101 +++++++++++ tests/functionaltests/suites/sru/default.cfg | 90 ---------- tests/functionaltests/suites/sru/default.yml | 100 +++++++++++ .../suites/stac_api/conftest.py | 161 +++++++++++------ .../functionaltests/suites/utf-8/default.cfg | 89 ---------- .../functionaltests/suites/utf-8/default.yml | 100 +++++++++++ .../utf-8/expected/post_GetCapabilities.xml | 22 +-- tests/functionaltests/suites/xslt/default.cfg | 94 ---------- tests/functionaltests/suites/xslt/default.yml | 108 ++++++++++++ .../test_xml_suites_functional.py | 4 +- tests/gen_html.py | 4 +- tests/unittests/test_server.py | 17 +- tests/unittests/test_wsgi.py | 33 ++-- 215 files changed, 4204 insertions(+), 3722 deletions(-) delete mode 100644 default-sample.cfg create mode 100644 default-sample.yml delete mode 100644 docker/compose/pycsw.cfg create mode 100644 docker/compose/pycsw.yml delete mode 100644 docker/pycsw.cfg create mode 100644 docker/pycsw.yml delete mode 100644 tests/functionaltests/suites/apiso-inspire/default.cfg create mode 100644 tests/functionaltests/suites/apiso-inspire/default.yml delete mode 100644 tests/functionaltests/suites/apiso/default.cfg create mode 100644 tests/functionaltests/suites/apiso/default.yml delete mode 100644 tests/functionaltests/suites/atom/default.cfg create mode 100644 tests/functionaltests/suites/atom/default.yml delete mode 100644 tests/functionaltests/suites/cite/default.cfg create mode 100644 tests/functionaltests/suites/cite/default.yml delete mode 100644 tests/functionaltests/suites/csw30/default.cfg create mode 100644 tests/functionaltests/suites/csw30/default.yml delete mode 100644 tests/functionaltests/suites/default/default.cfg create mode 100644 tests/functionaltests/suites/default/default.yml delete mode 100644 tests/functionaltests/suites/dif/default.cfg create mode 100644 tests/functionaltests/suites/dif/default.yml delete mode 100644 tests/functionaltests/suites/duplicatefileid/default.cfg create mode 100644 tests/functionaltests/suites/duplicatefileid/default.yml delete mode 100644 tests/functionaltests/suites/ebrim/default.cfg create mode 100644 tests/functionaltests/suites/ebrim/default.yml delete mode 100644 tests/functionaltests/suites/fgdc/default.cfg create mode 100644 tests/functionaltests/suites/fgdc/default.yml delete mode 100644 tests/functionaltests/suites/gm03/default.cfg create mode 100644 tests/functionaltests/suites/gm03/default.yml delete mode 100644 tests/functionaltests/suites/harvesting/default.cfg create mode 100644 tests/functionaltests/suites/harvesting/default.yml delete mode 100644 tests/functionaltests/suites/idswithpaths/default.cfg create mode 100644 tests/functionaltests/suites/idswithpaths/default.yml delete mode 100644 tests/functionaltests/suites/manager/default.cfg create mode 100644 tests/functionaltests/suites/manager/default.yml delete mode 100644 tests/functionaltests/suites/oaipmh/default.cfg create mode 100644 tests/functionaltests/suites/oaipmh/default.yml delete mode 100644 tests/functionaltests/suites/opensearcheo/default.cfg create mode 100644 tests/functionaltests/suites/opensearcheo/default.yml delete mode 100644 tests/functionaltests/suites/repofilter/default.cfg create mode 100644 tests/functionaltests/suites/repofilter/default.yml delete mode 100644 tests/functionaltests/suites/sru/default.cfg create mode 100644 tests/functionaltests/suites/sru/default.yml delete mode 100644 tests/functionaltests/suites/utf-8/default.cfg create mode 100644 tests/functionaltests/suites/utf-8/default.yml delete mode 100644 tests/functionaltests/suites/xslt/default.cfg create mode 100644 tests/functionaltests/suites/xslt/default.yml diff --git a/default-sample.cfg b/default-sample.cfg deleted file mode 100644 index ec3113dcd..000000000 --- a/default-sample.cfg +++ /dev/null @@ -1,101 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# -# Copyright (c) 2015 Tom Kralidis -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -[server] -home=/var/www/pycsw -url=http://localhost/pycsw/csw.py -mimetype=application/xml; charset=UTF-8 -encoding=UTF-8 -language=en-US -maxrecords=10 -#loglevel=DEBUG -#logfile=/tmp/pycsw.log -#ogc_schemas_base=http://foo -#federatedcatalogues=http://catalog.data.gov/csw -#pretty_print=true -gzip_compresslevel=9 -#domainquerytype=range -#domaincounts=true -#spatial_ranking=true -profiles=apiso -#workers=2 - -[manager] -transactions=false -allowed_ips=127.0.0.1 -#csw_harvest_pagesize=10 - -[metadata:main] -identification_title=pycsw Geospatial Catalogue -identification_abstract=pycsw is an OARec and OGC CSW server implementation written in Python -identification_keywords=catalogue,discovery,metadata -identification_keywords_type=theme -identification_fees=None -identification_accessconstraints=None -provider_name=Organization Name -provider_url=https://pycsw.org/ -contact_name=Lastname, Firstname -contact_position=Position Title -contact_address=Mailing Address -contact_city=City -contact_stateorprovince=Administrative Area -contact_postalcode=Zip or Postal Code -contact_country=Country -contact_phone=+xx-xxx-xxx-xxxx -contact_fax=+xx-xxx-xxx-xxxx -contact_email=you@example.org -contact_url=Contact URL -contact_hours=Hours of Service -contact_instructions=During hours of service. Off on weekends. -contact_role=pointOfContact - -[repository] -# sqlite -database=sqlite:////var/www/pycsw/tests/functionaltests/suites/cite/data/cite.db -# postgres -#database=postgresql://username:password@localhost/pycsw -# mysql -#database=mysql://username:password@localhost/pycsw?charset=utf8 -#mappings=path/to/mappings.py -table=records -#filter=type = 'http://purl.org/dc/dcmitype/Dataset' -#max_retries=5 -facets=type,title - -[metadata:inspire] -enabled=true -languages_supported=eng,gre -default_language=eng -date=YYYY-MM-DD -gemet_keywords=Utility and governmental services -conformity_service=notEvaluated -contact_name=Organization Name -contact_email=Email Address -temp_extent=YYYY-MM-DD/YYYY-MM-DD - diff --git a/default-sample.yml b/default-sample.yml new file mode 100644 index 000000000..6e66e6140 --- /dev/null +++ b/default-sample.yml @@ -0,0 +1,125 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Angelos Tzotsos +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2024 Angelos Tzotsos +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + url: http://localhost/pycsw/csw.py + mimetype: application/xml; charset=UTF-8 + encoding: UTF-8 + language: en-US + maxrecords: 10 + #ogc_schemas_location: http://foo + #pretty_print: true + gzip_compresslevel: 9 + #domainquerytype: range + #domaincounts: true + #spatial_ranking: true + #workers=2 + +logging: + level: ERROR + #logfile: /tmp/pycsw.log + +profiles: + - apiso + +federatedcatalogues: + - http://catalog.data.gov/csw + +manager: + transactions: false + allowed_ips: + - 127.0.0.1 + csw_harvest_pagesize: 10 + +metadata: + identification: + title: pycsw Geospatial Catalogue + description: pycsw is an OARec and OGC CSW server implementation written in Python + keywords: + - catalogue + - discovery + - metadata + keywords_type: theme + fees: None + accessconstraints: None + terms_of_service: https://creativecommons.org/licenses/by/4.0 + url: https://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0 + provider: + name: Organization Name + url: https://pycsw.org + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact + inspire: + enabled: true + languages_supported: + - eng + - gre + default_language: eng + date: YYYY-MM-DD + gemet_keywords: + - Utility and governmental services + conformity_service: notEvaluated + contact_name: Organization Name + contact_email: Email Address + temp_extent: + begin: YYYY-MM-DD + end: YYYY-MM-DD + +repository: + # sqlite + database: sqlite:////var/www/pycsw/tests/functionaltests/suites/cite/data/cite.db + # postgres + #database: postgresql://username:password@localhost/pycsw + # mysql + #database: mysql://username:password@localhost/pycsw?charset=utf8 + #mappings: path/to/mappings.py + table: records + #filter: type = 'http://purl.org/dc/dcmitype/Dataset' + #max_retries: 5 + facets: + - type + - title diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index 15e2b5969..a6fceb4e6 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -67,7 +67,7 @@ services: ports: - 8000:8000 volumes: - - ./pycsw.cfg:/etc/pycsw/pycsw.cfg + - ./pycsw.yml:/etc/pycsw/pycsw.yml volumes: diff --git a/docker/compose/pycsw.cfg b/docker/compose/pycsw.cfg deleted file mode 100644 index b1687a1b9..000000000 --- a/docker/compose/pycsw.cfg +++ /dev/null @@ -1,102 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# Ricardo Garcia Silva -# -# Copyright (c) 2015 Tom Kralidis -# Copyright (c) 2017 Ricardo Garcia Silva -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -[server] -home=/home/pycsw -url=${PYCSW_SERVER_URL} -mimetype=application/xml; charset=UTF-8 -encoding=UTF-8 -language=en-US -maxrecords=10 -loglevel=DEBUG -#logfile= -#ogc_schemas_base=http://foo -#federatedcatalogues=http://catalog.data.gov/csw -#pretty_print=true -#gzip_compresslevel=8 -#domainquerytype=range -#domaincounts=true -#spatial_ranking=true -profiles=apiso -#workers=2 -timeout=30 - -[manager] -transactions=false -allowed_ips=127.0.0.1 -#csw_harvest_pagesize=10 - -[metadata:main] -identification_title=pycsw Geospatial Catalogue -identification_abstract=pycsw is an OARec and OGC CSW server implementation written in Python -identification_keywords=catalogue,discovery,metadata -identification_keywords_type=theme -identification_fees=None -identification_accessconstraints=None -provider_name=Organization Name -provider_url=https://pycsw.org/ -contact_name=Lastname, Firstname -contact_position=Position Title -contact_address=Mailing Address -contact_city=City -contact_stateorprovince=Administrative Area -contact_postalcode=Zip or Postal Code -contact_country=Country -contact_phone=+xx-xxx-xxx-xxxx -contact_fax=+xx-xxx-xxx-xxxx -contact_email=Email Address -contact_url=Contact URL -contact_hours=Hours of Service -contact_instructions=During hours of service. Off on weekends. -contact_role=pointOfContact - -[repository] -# sqlite -#database=sqlite:////home/pycsw/tests/functionaltests/suites/cite/data/cite.db -# postgres -database=postgresql://postgres:mypass@db/pycsw -# mysql -#database=mysql://username:password@localhost/pycsw?charset=utf8 -#mappings=path/to/mappings.py -table=records -#filter=type = 'http://purl.org/dc/dcmitype/Dataset' - -[metadata:inspire] -enabled=true -languages_supported=eng,gre -default_language=eng -date=YYYY-MM-DD -gemet_keywords=Utility and governmental services -conformity_service=notEvaluated -contact_name=Organization Name -contact_email=Email Address -temp_extent=YYYY-MM-DD/YYYY-MM-DD - diff --git a/docker/compose/pycsw.yml b/docker/compose/pycsw.yml new file mode 100644 index 000000000..84879e567 --- /dev/null +++ b/docker/compose/pycsw.yml @@ -0,0 +1,114 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2017 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + url: ${PYCSW_SERVER_URL} + mimetype: application/xml; charset=UTF-8 + encoding: UTF-8 + language: en-US + maxrecords: 10 + timeout: 30 + #ogc_schemas_location: http://foo + #pretty_print: true + gzip_compresslevel: 9 + #domainquerytype: range + #domaincounts: true + #spatial_ranking: true + #workers=2 + +logging: + level: DEBUG + #logfile: /tmp/pycsw.log + +profiles: + - apiso + +federatedcatalogues: + - http://catalog.data.gov/csw + +manager: + transactions: false + allowed_ips: + - 127.0.0.1 +# csw_harvest_pagesize: 10 + +metadata: + identification: + title: pycsw Geospatial Catalogue + description: pycsw is an OARec and OGC CSW server implementation written in Python + keywords: + - catalogue + - discovery + - metadata + keywords_type: theme + fees: None + accessconstraints: None + terms_of_service: https://creativecommons.org/licenses/by/4.0 + url: https://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0 + provider: + name: Organization Name + url: https://pycsw.org + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact + + inspire: + enabled: true + languages_supported: + - eng + - gre + default_language: eng + date: YYYY-MM-DD + gemet_keywords: + - Utility and governmental services + conformity_service: notEvaluated + contact_name: Organization Name + contact_email=Email Address + temp_extent=YYYY-MM-DD/YYYY-MM-DD + +repository: + database: 'postgresql://postgres:mypass@db/pycsw' + table: records diff --git a/docker/entrypoint.py b/docker/entrypoint.py index d99ff6ac0..38b35d74f 100644 --- a/docker/entrypoint.py +++ b/docker/entrypoint.py @@ -5,7 +5,7 @@ # Tom Kralidis # # Copyright (c) 2017 Ricardo Garcia Silva -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -43,11 +43,9 @@ import argparse import logging import os -import configparser import sys from pycsw.core.config import StaticContext -from pycsw.core.util import EnvInterpolation from pycsw.core.repository import Repository, setup LOGGER = logging.getLogger(__name__) @@ -73,15 +71,15 @@ def launch_pycsw(pycsw_config, workers=2, reload=False): for more information on how to control gunicorn by sending UNIX signals. """ + database = pycsw_config['repository'].get('database') + table = pycsw_config['repository'].get('table') + try: - setup(pycsw_config.get("repository", "database"), - pycsw_config.get("repository", "table")) + setup(database, table) except Exception as err: LOGGER.debug(err) - repo = Repository(pycsw_config.get("repository", "database"), - StaticContext(), - table=pycsw_config.get("repository", "table")) + repo = Repository(database, StaticContext(), table=table) repo.ping() @@ -92,10 +90,7 @@ def launch_pycsw(pycsw_config, workers=2, reload=False): # https://github.com/benoitc/gunicorn/issues/1477 # - try: - timeout = pycsw_config.get("server", "timeout") - except configparser.NoOptionError: - timeout = 30 + timeout = pycsw_config["server"].get('timeout', 30) args = ["--reload", "--reload-engine=poll"] if reload else [] execution_args = ["gunicorn"] + args + [ @@ -129,16 +124,14 @@ def launch_pycsw(pycsw_config, workers=2, reload=False): "code changes? This option is only useful for development. " "Defaults to False." ) + args = parser.parse_args() - config = configparser.ConfigParser(interpolation=EnvInterpolation()) - config.read(os.getenv("PYCSW_CONFIG")) - try: - level = config.get("server", "loglevel").upper() - except configparser.NoOptionError: - level = "WARNING" - try: - workers = int(config.get("server", "workers")) - except configparser.NoOptionError: - workers = args.workers + + with open(os.getenv('PYCSW_CONFIG', encoding='utf8') as fh: + config = yaml_load(fh) + + level = config['logging'].get('level', 'WARNING') + + workers = int(config['server'].get('workers', args.workers)) logging.basicConfig(level=getattr(logging, level)) launch_pycsw(config, workers=workers, reload=args.reload) diff --git a/docker/helm/templates/pycsw-configmap.yaml b/docker/helm/templates/pycsw-configmap.yaml index 3192dd6e6..90e21c522 100644 --- a/docker/helm/templates/pycsw-configmap.yaml +++ b/docker/helm/templates/pycsw-configmap.yaml @@ -1,71 +1,85 @@ apiVersion: v1 data: - pycsw.cfg: |+ - [server] -{{ if .Values.pycsw.config.server.home }} home={{ .Values.pycsw.config.server.home }} -{{ end }}{{ if .Values.pycsw.config.server.url }} url={{ .Values.pycsw.config.server.url }} -{{ end }}{{ if .Values.pycsw.config.server.mimetype }} mimetype={{ .Values.pycsw.config.server.mimetype }} -{{ end }}{{ if .Values.pycsw.config.server.encoding }} encoding={{ .Values.pycsw.config.server.encoding }} -{{ end }}{{ if .Values.pycsw.config.server.language }} language={{ .Values.pycsw.config.server.language }} -{{ end }}{{ if .Values.pycsw.config.server.maxrecords }} maxrecords={{ .Values.pycsw.config.server.maxrecords }} -{{ end }}{{ if .Values.pycsw.config.server.loglevel }} loglevel={{ .Values.pycsw.config.server.loglevel }} -{{ end }}{{ if .Values.pycsw.config.server.logfile }} logfile={{ .Values.pycsw.config.server.logfile }} -{{ end }}{{ if .Values.pycsw.config.server.ogc_schemas_base }} ogc_schemas_base={{ .Values.pycsw.config.server.ogc_schemas_base }} -{{ end }}{{ if .Values.pycsw.config.server.federatedcatalogues }} federatedcatalogues={{ .Values.pycsw.config.server.federatedcatalogues }} -{{ end }}{{ if .Values.pycsw.config.server.pretty_print }} pretty_print={{ .Values.pycsw.config.server.pretty_print }} -{{ end }}{{ if .Values.pycsw.config.server.gzip_compresslevel }} gzip_compresslevel={{ .Values.pycsw.config.server.gzip_compresslevel }} -{{ end }}{{ if .Values.pycsw.config.server.domainquerytype }} domainquerytype={{ .Values.pycsw.config.server.domainquerytype }} -{{ end }}{{ if .Values.pycsw.config.server.domaincounts }} domaincounts={{ .Values.pycsw.config.server.domaincounts }} -{{ end }}{{ if .Values.pycsw.config.server.spatial_ranking }} spatial_ranking={{ .Values.pycsw.config.server.spatial_ranking }} -{{ end }}{{ if .Values.pycsw.config.server.profiles }} profiles={{ .Values.pycsw.config.server.profiles }} -{{ end }}{{ if .Values.pycsw.config.server.workers }} workers={{ .Values.pycsw.config.server.workers }} -{{ end }}{{ if .Values.pycsw.config.server.timeout }} timeout={{ .Values.pycsw.config.server.timeout }} + pycsw.yml: |+ + server: +{{ if .Values.pycsw.config.server.home }} home: {{ .Values.pycsw.config.server.home }} +{{ end }}{{ if .Values.pycsw.config.server.url }} url: {{ .Values.pycsw.config.server.url }} +{{ end }}{{ if .Values.pycsw.config.server.mimetype }} mimetype: {{ .Values.pycsw.config.server.mimetype }} +{{ end }}{{ if .Values.pycsw.config.server.encoding }} encoding:{{ .Values.pycsw.config.server.encoding }} +{{ end }}{{ if .Values.pycsw.config.server.language }} language: {{ .Values.pycsw.config.server.language }} +{{ end }}{{ if .Values.pycsw.config.server.maxrecords }} maxrecords: {{ .Values.pycsw.config.server.maxrecords }} +{{ end }}{{ if .Values.pycsw.config.server.ogc_schemas_base }} ogc_schemas_base: {{ .Values.pycsw.config.server.ogc_schemas_base }} +{{ end }}{{ if .Values.pycsw.config.server.federatedcatalogues }} federatedcatalogues: {{ .Values.pycsw.config.server.federatedcatalogues }} +{{ end }}{{ if .Values.pycsw.config.server.pretty_print }} pretty_print: {{ .Values.pycsw.config.server.pretty_print }} +{{ end }}{{ if .Values.pycsw.config.server.gzip_compresslevel }} gzip_compresslevel: {{ .Values.pycsw.config.server.gzip_compresslevel }} +{{ end }}{{ if .Values.pycsw.config.server.domainquerytype }} domainquerytype: {{ .Values.pycsw.config.server.domainquerytype }} +{{ end }}{{ if .Values.pycsw.config.server.domaincounts }} domaincounts: {{ .Values.pycsw.config.server.domaincounts }} +{{ end }}{{ if .Values.pycsw.config.server.spatial_ranking }} spatial_ranking: {{ .Values.pycsw.config.server.spatial_ranking }} +{{ end }}{{ if .Values.pycsw.config.server.workers }} workers: {{ .Values.pycsw.config.server.workers }} +{{ end }}{{ if .Values.pycsw.config.server.timeout }} timeout: {{ .Values.pycsw.config.server.timeout }} {{ end }} - [manager] -{{ if .Values.pycsw.config.manager.transactions }} transactions={{ .Values.pycsw.config.manager.transactions }} -{{ end }}{{ if .Values.pycsw.config.manager.allowed_ips }} allowed_ips={{ .Values.pycsw.config.manager.allowed_ips }} -{{ end }}{{ if .Values.pycsw.config.manager.csw_harvest_pagesize }} csw_harvest_pagesize={{ .Values.pycsw.config.manager.csw_harvest_pagesize }} + +{{ if .Values.pycsw.config.profiles }} + profiles: + {{ .Values.pycsw.config.profiles }} +{{ end }} + +{{ if .Values.pycsw.config.logging.level }} + logging: + level:{{ .Values.pycsw.config.logging.level }} +{{ if .Values.pycsw.config.logging.logfile }} logfile: {{ .Values.pycsw.config.logging.logfile }} +{{ end }} + + manager: +{{ if .Values.pycsw.config.manager.transactions }} transactions: {{ .Values.pycsw.config.manager.transactions }} +{{ end }}{{ if .Values.pycsw.config.manager.allowed_ips }} allowed_ips: {{ .Values.pycsw.config.manager.allowed_ips }} +{{ end }}{{ if .Values.pycsw.config.manager.csw_harvest_pagesize }} csw_harvest_pagesize: {{ .Values.pycsw.config.manager.csw_harvest_pagesize }} +{{ end }} + metadata: + identification: +{{ if .Values.pycsw.config.metadata.identification.title }} title: {{ .Values.pycsw.config.metadata.identification.title }} +{{ end }}{{ if .Values.pycsw.config.metadata.identification.description }} description: {{ .Values.pycsw.config.metadata.identification.description }} +{{ end }}{{ if .Values.pycsw.config.metadata.identification.keywords }} keywords: {{ .Values.pycsw.config.metadata.identification.keywords }} +{{ end }}{{ if .Values.pycsw.config.metadata.identification.keywords_type }} keywords_type: {{ .Values.pycsw.config.metadata.identification.keywords_type }} +{{ end }}{{ if .Values.pycsw.config.metadata.identification.fees }} fees: {{ .Values.pycsw.config.metadata.identification.fees }} +{{ end }}{{ if .Values.pycsw.config.metadata.identification.accessconstraints }} accessconstraints: {{ .Values.pycsw.config.metadata.identification.accessconstraints }} +{{ end }} + provider: +{{ end }}{{ if .Values.pycsw.config.metadata.provider.name }} name: {{ .Values.pycsw.config.metadata.provider.name }} +{{ end }}{{ if .Values.pycsw.config.metadata.provider_url }} url: {{ .Values.pycsw.config.metadata.provider.url }} {{ end }} - [metadata:main] -{{ if .Values.pycsw.config.metadata.identification_title }} identification_title={{ .Values.pycsw.config.metadata.identification_title }} -{{ end }}{{ if .Values.pycsw.config.metadata.identification_abstract }} identification_abstract={{ .Values.pycsw.config.metadata.identification_abstract }} -{{ end }}{{ if .Values.pycsw.config.metadata.identification_keywords }} identification_keywords={{ .Values.pycsw.config.metadata.identification_keywords }} -{{ end }}{{ if .Values.pycsw.config.metadata.identification_keywords_type }} identification_keywords_type={{ .Values.pycsw.config.metadata.identification_keywords_type }} -{{ end }}{{ if .Values.pycsw.config.metadata.identification_fees }} identification_fees={{ .Values.pycsw.config.metadata.identification_fees }} -{{ end }}{{ if .Values.pycsw.config.metadata.identification_accessconstraints }} identification_accessconstraints={{ .Values.pycsw.config.metadata.identification_accessconstraints }} -{{ end }}{{ if .Values.pycsw.config.metadata.provider_name }} provider_name={{ .Values.pycsw.config.metadata.provider_name }} -{{ end }}{{ if .Values.pycsw.config.metadata.provider_url }} provider_url={{ .Values.pycsw.config.metadata.provider_url }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_name }} contact_name={{ .Values.pycsw.config.metadata.contact_name }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_position }} contact_position={{ .Values.pycsw.config.metadata.contact_position }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_address }} contact_address={{ .Values.pycsw.config.metadata.contact_address }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_city }} contact_city={{ .Values.pycsw.config.metadata.contact_city }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_stateorprovince }} contact_stateorprovince={{ .Values.pycsw.config.metadata.contact_stateorprovince }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_postalcode }} contact_postalcode={{ .Values.pycsw.config.metadata.contact_postalcode }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_country }} contact_country={{ .Values.pycsw.config.metadata.contact_country }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_phone }} contact_phone={{ .Values.pycsw.config.metadata.contact_phone }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_fax }} contact_fax={{ .Values.pycsw.config.metadata.contact_fax }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_email }} contact_email={{ .Values.pycsw.config.metadata.contact_email }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_url }} contact_url={{ .Values.pycsw.config.metadata.contact_url }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_hours }} contact_hours={{ .Values.pycsw.config.metadata.contact_hours }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_instructions }} contact_instructions={{ .Values.pycsw.config.metadata.contact_instructions }} -{{ end }}{{ if .Values.pycsw.config.metadata.contact_role }} contact_role={{ .Values.pycsw.config.metadata.contact_role }} + contact: +{{ end }}{{ if .Values.pycsw.config.metadata.contact.name }} name: {{ .Values.pycsw.config.metadata.contact.name }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.position }} position: {{ .Values.pycsw.config.metadata.contact.position }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.address }} address: {{ .Values.pycsw.config.metadata.contact.address }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.city }} city: {{ .Values.pycsw.config.metadata.contact.city }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.stateorprovince }} stateorprovince: {{ .Values.pycsw.config.metadata.contact.stateorprovince }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.postalcode }} postalcode: {{ .Values.pycsw.config.metadata.contact.postalcode }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.country }} country: {{ .Values.pycsw.config.metadata.contact.country }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.phone }} phone: {{ .Values.pycsw.config.metadata.contact.phone }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.fax }} fax: {{ .Values.pycsw.config.metadata.contact.fax }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.email }} email: {{ .Values.pycsw.config.metadata.contact.email }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.url }} url: {{ .Values.pycsw.config.metadata.contact.url }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.hours }} hours: {{ .Values.pycsw.config.metadata.contact.hours }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.instructions }} instructions: {{ .Values.pycsw.config.metadata.contact.instructions }} +{{ end }}{{ if .Values.pycsw.config.metadata.contact.role }} role: {{ .Values.pycsw.config.metadata.contact.role }} {{ end }} - [repository] -{{ if .Values.pycsw.config.repository.database }} database={{ .Values.pycsw.config.repository.database }} -{{ end }}{{ if .Values.pycsw.config.repository.mappings }} mappings={{ .Values.pycsw.config.repository.mappings }} -{{ end }}{{ if .Values.pycsw.config.repository.table }} table={{ .Values.pycsw.config.repository.table }} -{{ end }}{{ if .Values.pycsw.config.repository.filter }} filter={{ .Values.pycsw.config.repository.filter }} + inspire: +{{ if .Values.pycsw.config.metadata.inspire.enabled }} enabled: {{ .Values.pycsw.config.metadata.inspire.enabled }} +{{ end }}{{ if .Values.pycsw.config.metadata.inspire.languages_supported }} languages_supported: {{ .Values.pycsw.config.metadata.inspire.languages_supported }} +{{ end }}{{ if .Values.pycsw.config.metadata.inspire.default_language }} default_language: {{ .Values.pycsw.config.metadata.inspire.default_language }} +{{ end }}{{ if .Values.pycsw.config.metadata.inspire.date }} date: {{ .Values.pycsw.config.metadata.inspire.date }} +{{ end }}{{ if .Values.pycsw.config.inspire.gemet_keywords }} gemet_keywords: {{ .Values.pycsw.config.metadata.inspire.gemet_keywords }} +{{ end }}{{ if .Values.pycsw.config.inspire.conformity_service }} conformity_service: {{ .Values.pycsw.config.metadata.inspire.conformity_service }} +{{ end }}{{ if .Values.pycsw.config.inspire.contact_name }} contact_name: {{ .Values.pycsw.config.metadata.inspire.contact_name }} +{{ end }}{{ if .Values.pycsw.config.inspire.contact_email }} contact_email: {{ .Values.pycsw.config.metadata.inspire.contact_email }} +{{ end }}{{ if .Values.pycsw.config.inspire.temp_extent }} temp_extent: {{ .Values.pycsw.config.metadata.inspire.temp_extent }} {{ end }} - [metadata:inspire] -{{ if .Values.pycsw.config.inspire.enabled }} enabled={{ .Values.pycsw.config.inspire.enabled }} -{{ end }}{{ if .Values.pycsw.config.inspire.languages_supported }} languages_supported={{ .Values.pycsw.config.inspire.languages_supported }} -{{ end }}{{ if .Values.pycsw.config.inspire.default_language }} default_language={{ .Values.pycsw.config.inspire.default_language }} -{{ end }}{{ if .Values.pycsw.config.inspire.date }} date={{ .Values.pycsw.config.inspire.date }} -{{ end }}{{ if .Values.pycsw.config.inspire.gemet_keywords }} gemet_keywords={{ .Values.pycsw.config.inspire.gemet_keywords }} -{{ end }}{{ if .Values.pycsw.config.inspire.conformity_service }} conformity_service={{ .Values.pycsw.config.inspire.conformity_service }} -{{ end }}{{ if .Values.pycsw.config.inspire.contact_name }} contact_name={{ .Values.pycsw.config.inspire.contact_name }} -{{ end }}{{ if .Values.pycsw.config.inspire.contact_email }} contact_email={{ .Values.pycsw.config.inspire.contact_email }} -{{ end }}{{ if .Values.pycsw.config.inspire.temp_extent }} temp_extent={{ .Values.pycsw.config.inspire.temp_extent }} + repository: +{{ if .Values.pycsw.config.repository.database }} database: {{ .Values.pycsw.config.repository.database }} +{{ end }}{{ if .Values.pycsw.config.repository.mappings }} mappings: {{ .Values.pycsw.config.repository.mappings }} +{{ end }}{{ if .Values.pycsw.config.repository.table }} table: {{ .Values.pycsw.config.repository.table }} +{{ end }}{{ if .Values.pycsw.config.repository.filter }} filter: {{ .Values.pycsw.config.repository.filter }} {{ end }} kind: ConfigMap diff --git a/docker/helm/values.yaml b/docker/helm/values.yaml index 36713e4f2..d78f2d5a0 100644 --- a/docker/helm/values.yaml +++ b/docker/helm/values.yaml @@ -35,7 +35,6 @@ pycsw: encoding: UTF-8 language: en-US maxrecords: 10 - loglevel: DEBUG # logfile: " " # ogc_schemas_base: http://foo # federatedcatalogues: http://catalog.data.gov/csw @@ -44,49 +43,63 @@ pycsw: # domainquerytype: range # domaincounts: true # spatial_ranking: true - profiles: apiso # workers: 2 timeout: 30 + profiles: + - apiso + logging: + level: DEBUG manager: - transactions: "false" - allowed_ips: 127.0.0.1 + transactions: false + allowed_ips: + - 127.0.0.1 # csw_harvest_pagesize: 10 metadata: - identification_title: pycsw Geospatial Catalogue - identification_abstract: pycsw is an OARec and OGC CSW server implementation written in Python - identification_keywords: catalogue,discovery,metadata - identification_keywords_type: theme - identification_fees: None - identification_accessconstraints: None - provider_name: Organization Name - provider_url: https://pycsw.org/ - contact_name: Lastname, Firstname - contact_position: Position Title - contact_address: Mailing Address - contact_city: City - contact_stateorprovince: Administrative Area - contact_postalcode: Zip or Postal Code - contact_country: Country - contact_phone: +xx-xxx-xxx-xxxx - contact_fax: +xx-xxx-xxx-xxxx - contact_email: Email Address - contact_url: Contact URL - contact_hours: Hours of Service - contact_instructions: During hours of service. Off on weekends. - contact_role: pointOfContact + identification: + title: pycsw Geospatial Catalogue + description: pycsw is an OARec and OGC CSW server implementation written in Python + keywords: + - catalogue + - discovery + - metadata + keywords_type: theme + fees: None + accessconstraints: None + provider: + name: Organization Name + url: https://pycsw.org/ + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: Email Address + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + inspire: + enabled: true + languages_supported: + - eng + - gre + default_language: eng + date: YYYY-MM-DD + gemet_keywords: + - Utility and governmental services + conformity_service: notEvaluated + contact_name: Organization Name + contact_email: Email Address + temp_extent: + begin: YYYY-MM-DD + end: YYYY-MM-DD repository: database: postgresql://postgres:mypass@db/pycsw table: records # mappings: path/to/mappings.py # filter: type = 'http://purl.org/dc/dcmitype/Dataset' - inspire: - enabled: "true" - languages_supported: eng,gre - default_language: eng - date: YYYY-MM-DD - gemet_keywords: Utility and governmental services - conformity_service: notEvaluated - contact_name: Organization Name - contact_email: Email Address - temp_extent: YYYY-MM-DD/YYYY-MM-DD - diff --git a/docker/kubernetes/pycsw-configmap.yaml b/docker/kubernetes/pycsw-configmap.yaml index 8fce3cd5c..1b854fc42 100644 --- a/docker/kubernetes/pycsw-configmap.yaml +++ b/docker/kubernetes/pycsw-configmap.yaml @@ -1,13 +1,15 @@ apiVersion: v1 data: - pycsw.cfg: |+ + pycsw.yml: |+ # ================================================================= # # Authors: Tom Kralidis # Ricardo Garcia Silva + # Angelos Tzotsos # - # Copyright (c) 2015 Tom Kralidis + # Copyright (c) 2024 Tom Kralidis # Copyright (c) 2017 Ricardo Garcia Silva + # Copyright (c) 2024 Angelos Tzotsos # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -31,77 +33,94 @@ data: # OTHER DEALINGS IN THE SOFTWARE. # # ================================================================= + + server: + url: ${PYCSW_SERVER_URL} + mimetype: application/xml; charset=UTF-8 + encoding: UTF-8 + language: en-US + maxrecords: 10 + timeout: 30 + #ogc_schemas_location: http://foo + #pretty_print: true + #gzip_compresslevel: 9 + #domainquerytype: range + #domaincounts: true + #spatial_ranking: true + #workers=2 + + logging: + level: DEBUG + #logfile: /tmp/pycsw.log + + profiles: + - apiso + + federatedcatalogues: + - http://catalog.data.gov/csw + + manager: + transactions: false + allowed_ips: + - 127.0.0.1 + #csw_harvest_pagesize: 10 + + metadata: + identification: + title: pycsw Geospatial Catalogue + description: pycsw is an OARec and OGC CSW server implementation written in Python + keywords: + - catalogue + - discovery + - metadata + keywords_type: theme + fees: None + accessconstraints: None + terms_of_service: https://creativecommons.org/licenses/by/4.0 + url: https://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0 + provider: + name: Organization Name + url: https://pycsw.org + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact + inspire: + enabled: true + languages_supported: + - eng + - gre + default_language: eng + date: YYYY-MM-DD + gemet_keywords: + - Utility and governmental services + conformity_service: notEvaluated + contact_name: Organization Name + contact_email: Email Address + temp_extent: + begin: YYYY-MM-DD + end: YYYY-MM-DD - [server] - home=/home/pycsw - url=${PYCSW_SERVER_URL} - mimetype=application/xml; charset=UTF-8 - encoding=UTF-8 - language=en-US - maxrecords=10 - loglevel=DEBUG - #logfile= - #ogc_schemas_base=http://foo - #federatedcatalogues=http://catalog.data.gov/csw - #pretty_print=true - #gzip_compresslevel=8 - #domainquerytype=range - #domaincounts=true - #spatial_ranking=true - profiles=apiso - #workers=2 - timeout=30 - - [manager] - transactions=false - allowed_ips=127.0.0.1 - #csw_harvest_pagesize=10 - - [metadata:main] - identification_title=pycsw Geospatial Catalogue - identification_abstract=pycsw is an OARec and OGC CSW server implementation written in Python - identification_keywords=catalogue,discovery,metadata - identification_keywords_type=theme - identification_fees=None - identification_accessconstraints=None - provider_name=Organization Name - provider_url=https://pycsw.org/ - contact_name=Lastname, Firstname - contact_position=Position Title - contact_address=Mailing Address - contact_city=City - contact_stateorprovince=Administrative Area - contact_postalcode=Zip or Postal Code - contact_country=Country - contact_phone=+xx-xxx-xxx-xxxx - contact_fax=+xx-xxx-xxx-xxxx - contact_email=Email Address - contact_url=Contact URL - contact_hours=Hours of Service - contact_instructions=During hours of service. Off on weekends. - contact_role=pointOfContact - - [repository] - # sqlite - #database=sqlite:////home/pycsw/tests/functionaltests/suites/cite/data/cite.db - # postgres - database=${PYCSW_REPOSITORY_DATABASE_URI} - # mysql - #database=mysql://username:password@localhost/pycsw?charset=utf8 - #mappings=path/to/mappings.py - table=records - #filter=type = 'http://purl.org/dc/dcmitype/Dataset' - - [metadata:inspire] - enabled=true - languages_supported=eng,gre - default_language=eng - date=YYYY-MM-DD - gemet_keywords=Utility and governmental services - conformity_service=notEvaluated - contact_name=Organization Name - contact_email=Email Address - temp_extent=YYYY-MM-DD/YYYY-MM-DD + repository: + database: ${PYCSW_REPOSITORY_DATABASE_URI} + table: records + facets: + - type + - title kind: ConfigMap metadata: diff --git a/docker/pycsw.cfg b/docker/pycsw.cfg deleted file mode 100644 index da6705f53..000000000 --- a/docker/pycsw.cfg +++ /dev/null @@ -1,102 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# Ricardo Garcia Silva -# -# Copyright (c) 2015 Tom Kralidis -# Copyright (c) 2017 Ricardo Garcia Silva -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -[server] -home=/home/pycsw -url=http://localhost:8000/ -mimetype=application/xml; charset=UTF-8 -encoding=UTF-8 -language=en-US -maxrecords=10 -loglevel=DEBUG -#logfile= -#ogc_schemas_base=http://foo -#federatedcatalogues=http://catalog.data.gov/csw -#pretty_print=true -#gzip_compresslevel=8 -#domainquerytype=range -#domaincounts=true -#spatial_ranking=true -profiles=apiso -#workers=2 -timeout=30 - -[manager] -transactions=false -allowed_ips=127.0.0.1 -#csw_harvest_pagesize=10 - -[metadata:main] -identification_title=pycsw Geospatial Catalogue -identification_abstract=pycsw is an OARec and OGC CSW server implementation written in Python -identification_keywords=catalogue,discovery,metadata -identification_keywords_type=theme -identification_fees=None -identification_accessconstraints=None -provider_name=Organization Name -provider_url=https://pycsw.org/ -contact_name=Lastname, Firstname -contact_position=Position Title -contact_address=Mailing Address -contact_city=City -contact_stateorprovince=Administrative Area -contact_postalcode=Zip or Postal Code -contact_country=Country -contact_phone=+xx-xxx-xxx-xxxx -contact_fax=+xx-xxx-xxx-xxxx -contact_email=Email Address -contact_url=Contact URL -contact_hours=Hours of Service -contact_instructions=During hours of service. Off on weekends. -contact_role=pointOfContact - -[repository] -# sqlite -database=sqlite:////home/pycsw/pycsw/tests/functionaltests/suites/cite/data/cite.db -# postgres -#database=postgresql://username:password@localhost/pycsw -# mysql -#database=mysql://username:password@localhost/pycsw?charset=utf8 -#mappings=path/to/mappings.py -table=records -#filter=type = 'http://purl.org/dc/dcmitype/Dataset' - -[metadata:inspire] -enabled=true -languages_supported=eng,gre -default_language=eng -date=YYYY-MM-DD -gemet_keywords=Utility and governmental services -conformity_service=notEvaluated -contact_name=Organization Name -contact_email=Email Address -temp_extent=YYYY-MM-DD/YYYY-MM-DD - diff --git a/docker/pycsw.yml b/docker/pycsw.yml new file mode 100644 index 000000000..57a014968 --- /dev/null +++ b/docker/pycsw.yml @@ -0,0 +1,115 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2017 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + url: http://localhost:8000/ + mimetype: application/xml; charset=UTF-8 + encoding: UTF-8 + language: en-US + maxrecords: 10 + timeout: 30 + #ogc_schemas_location: http://foo + #pretty_print: true + gzip_compresslevel: 9 + #domainquerytype: range + #domaincounts: true + #spatial_ranking: true + #workers=2 + +logging: + level: DEBUG + #logfile: /tmp/pycsw.log + +profiles: + - apiso + +federatedcatalogues: + - http://catalog.data.gov/csw + +manager: + transactions: false + allowed_ips: + - 127.0.0.1 +# csw_harvest_pagesize: 10 + +metadata: + identification: + title: pycsw Geospatial Catalogue + description: pycsw is an OARec and OGC CSW server implementation written in Python + keywords: + - catalogue + - discovery + - metadata + keywords_type: theme + fees: None + accessconstraints: None + terms_of_service: https://creativecommons.org/licenses/by/4.0 + url: https://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0 + provider: + name: Organization Name + url: https://pycsw.org + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Mo-Fr 08:00-17:00 + instructions: During hours of service. Off on weekends. + role: pointOfContact + + inspire: + enabled: true + languages_supported: + - eng + - gre + default_language: eng + date: YYYY-MM-DD + gemet_keywords: + - Utility and governmental services + conformity_service: notEvaluated + contact_name: Organization Name + contact_email=Email Address + temp_extent=YYYY-MM-DD/YYYY-MM-DD + +repository: + # sqlite + database: 'sqlite:////home/pycsw/pycsw/tests/functionaltests/suites/cite/data/cite.db' + table: records diff --git a/docs/administration.rst b/docs/administration.rst index 8009cf13a..7ffe4f75d 100644 --- a/docs/administration.rst +++ b/docs/administration.rst @@ -39,9 +39,9 @@ To expose your geospatial metadata via pycsw, perform the following actions: Supported Information Models ---------------------------- -By default, pycsw's API supports the core OARec and CSW Record information models. From +By default, pycsw's API supports the core OGC API - Records and CSW Record information models. From the database perspective, the pycsw metadata model is loosely based on ISO 19115 and is -able to transform to other formats as part of transformation during OARec/CSW requests. +able to transform to other formats as part of transformation during OGC API - Records/CSW requests. .. note:: See :ref:`profiles` for information on enabling profiles @@ -54,7 +54,7 @@ Setting up the Database .. code-block:: bash - pycsw-admin.py setup-repository --config default.cfg + pycsw-admin.py setup-repository --config default.yml This will create the necessary tables and values for the repository. @@ -78,9 +78,9 @@ Loading Records .. code-block:: bash - pycsw-admin.py load-records --config default.cfg --path /path/to/records + pycsw-admin.py load-records --config default.yml --path /path/to/records -This will import all ``*.xml`` records from ``/path/to/records`` into the database specified in ``default.cfg`` (``repository.database``). Passing ``-r`` to the script will process ``/path/to/records`` recursively. Passing ``-y`` to the script will force overwrite existing metadata with the same identifier. Note that ``-p`` accepts either a directory path or single file. +This will import all ``*.xml`` records from ``/path/to/records`` into the database specified in ``default.yml`` (``repository.database``). Passing ``-r`` to the script will process ``/path/to/records`` recursively. Passing ``-y`` to the script will force overwrite existing metadata with the same identifier. Note that ``-p`` accepts either a directory path or single file. .. note:: Records can also be imported using CSW-T (see :ref:`transactions`). @@ -90,17 +90,17 @@ Exporting the Repository .. code-block:: bash - pycsw-admin.py export-records --config default.cfg --path /path/to/output_dir + pycsw-admin.py export-records --config default.yml --path /path/to/output_dir -This will write each record in the database specified in ``default.cfg`` (``repository.database``) to an XML document on disk, in directory ``/path/to/output_dir``. +This will write each record in the database specified in ``default.yml`` (``repository.database``) to an XML document on disk, in directory ``/path/to/output_dir``. Optimizing the Database ----------------------- .. code-block:: bash - pycsw-admin.py optimize-db --config default.cfg - pycsw-admin.py rebuild-db-indexes --config default.cfg + pycsw-admin.py optimize-db --config default.yml + pycsw-admin.py rebuild-db-indexes --config default.yml .. note:: This feature is relevant only for PostgreSQL and MySQL @@ -110,7 +110,7 @@ Deleting Records from the Repository .. code-block:: bash - pycsw-admin.py delete-records --config default.cfg + pycsw-admin.py delete-records --config default.yml This will empty the repository of all records. @@ -121,7 +121,7 @@ PostgreSQL ^^^^^^^^^^ - To enable PostgreSQL support, the database user must be able to create functions within the database. -- `PostgreSQL Full Text Search`_ is supported for ``csw:AnyText`` based queries. pycsw creates a tsvector column based on the text from anytext column. Then pycsw creates a GIN index against the anytext_tsvector column. This is created automatically in ``pycsw.core.repository.setup``. Any query against OARec's ``q`` parameter or CSW `csw:AnyText` or `apiso:AnyText` will process using PostgreSQL FTS handling +- `PostgreSQL Full Text Search`_ is supported for ``csw:AnyText`` based queries. pycsw creates a tsvector column based on the text from anytext column. Then pycsw creates a GIN index against the anytext_tsvector column. This is created automatically in ``pycsw.core.repository.setup``. Any query against the OGC API - Records ``q`` parameter or CSW `csw:AnyText` or `apiso:AnyText` will process using PostgreSQL FTS handling PostGIS ^^^^^^^ @@ -145,21 +145,21 @@ mappings are in ``pycsw/core/config.py:StaticContext.md_core_model``). To override the default settings: - define a custom database mapping based on ``etc/mappings.py`` -- in ``default.cfg``, set ``repository.mappings`` to the location of the mappings.py file: +- in ``default.yml``, set ``repository.mappings`` to the location of the mappings.py file: -.. code-block:: none +.. code-block:: yaml - [repository] - ... - mappings=path/to/mappings.py + repository: + ... + mappings: path/to/mappings.py Note you can also reference mappings as a Python object as a dotted path: -.. code-block:: none +.. code-block:: yaml - [repository] - ... - mappings='path.to.pycsw_mappings' + repository: + ... + mappings: path.to.pycsw_mappings See the :ref:`geonode`, :ref:`hhypermap`, and :ref:`odc` for further examples. @@ -375,14 +375,14 @@ the DB, that the ``identifier`` column of the SQL view should be assumed to be t Finally, we can configure pycsw with the path to the custom mappings and the name of the SQL view: -.. code-block:: ini +.. code-block:: yaml - # file: pycsw.cfg + # file: pycsw.yml - [repository] - database=postgresql://${DB_USERNAME}:${DB_PASSWORD}@${DB_HOST}/${DB_NAME} - mappings=/path/to/my_custom_pycsw_mappings.py - table=my_pycsw_view + repository: + database: postgresql://${DB_USERNAME}:${DB_PASSWORD}@${DB_HOST}/${DB_NAME} + mappings: /path/to/my_custom_pycsw_mappings.py + table: my_pycsw_view .. _`GDAL`: https://www.gdal.org diff --git a/docs/api.rst b/docs/api.rst index c431536d2..c33b22d9f 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,8 +14,8 @@ services could be used: - authentication or authorization logic - forcing CSW version 2.0.2 as default -OARec Flask Example -------------------- +OGC API - Records Flask Example +------------------------------- See https://github.com/geopython/pycsw/blob/master/pycsw/wsgi_flask.py for how to implement a Flask wrapper atop all pycsw supported APIs. Note the use of @@ -66,8 +66,7 @@ Simple CSW Flask Example pycsw_config = some_dict # really comes from somewhere # initialize pycsw - # pycsw_config: either a ConfigParser object or a dict of - # the pycsw configuration + # pycsw_config: dict of the pycsw configuration # # env: dict of (HTTP) environment (defaults to os.environ) # diff --git a/docs/configuration.rst b/docs/configuration.rst index be4a7e926..e3448baa7 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -3,9 +3,9 @@ Configuration ============= -pycsw's runtime configuration is defined by ``default.cfg``. pycsw ships with a `sample configuration`_ (``default-sample.cfg``). Copy the file to ``default.cfg`` and edit the following: +pycsw's runtime configuration is defined by ``default.yml``. pycsw ships with a `sample configuration`_ (``default-sample.yml``). Copy the file to ``default.yml`` and edit the following: -**[server]** +**server** - **home**: the full filesystem path to pycsw - **url**: the URL of the resulting service @@ -13,7 +13,7 @@ pycsw's runtime configuration is defined by ``default.cfg``. pycsw ships with a - **language**: the ISO 639-1 language and ISO 3166-1 alpha2 country code of the service (e.g. ``en-CA``, ``fr-CA``, ``en-US``) - **encoding**: the content type encoding (e.g. ``ISO-8859-1``, see https://docs.python.org/2/library/codecs.html#standard-encodings). Default value is 'UTF-8' - **maxrecords**: the maximum number of records to return by default. This value is enforced if a CSW's client's ``maxRecords`` parameter is greater than ``server.maxrecords`` to limit capacity. See :ref:`maxrecords-handling` for more information -- **loglevel**: the logging level (see https://docs.python.org/library/logging.html#logging-levels) +- **level**: the logging level (see https://docs.python.org/library/logging.html#logging-levels) - **logfile**: the full file path to the logfile - **ogc_schemas_base**: base URL of OGC XML schemas tree file structure (default is http://schemas.opengis.net) - **federatedcatalogues**: comma delimited list of CSW endpoints to be used for distributed searching, if requested by the client (see :ref:`distributedsearching`) @@ -21,7 +21,6 @@ pycsw's runtime configuration is defined by ``default.cfg``. pycsw ships with a - **gzip_compresslevel**: gzip compression level, lowest is ``1``, highest is ``9``. Default is off. **NOTE**: if gzip compression is already enabled via your web server, do not enable this directive (or else the server will try to compress the response twice, resulting in degraded performance) - **domainquerytype**: for GetDomain operations, how to output domain values. Accepted values are ``list`` and ``range`` (min/max). Default is ``list`` - **domaincounts**: for GetDomain operations, whether to provide frequency counts for values. Accepted values are ``true`` and ``False``. Default is ``false`` -- **profiles**: comma delimited list of profiles to load at runtime (default is none). See :ref:`profiles` - **smtp_host**: SMTP host for processing ``csw:ResponseHandler`` parameter via outgoing email requests (default is ``localhost``) - **smtp_user**: SMTP user name related to the account (default is '') - **smtp_pass**: SMTP password related to the account (default is '') @@ -29,38 +28,50 @@ pycsw's runtime configuration is defined by ``default.cfg``. pycsw ships with a - **spatial_ranking**: parameter that enables (``true`` or ``false``) ranking of spatial query results as per `K.J. Lanfear 2006 - A Spatial Overlay Ranking Method for a Geospatial Search of Text Objects `_. - **workers**: set the number of workers used by the wsgi server when lunching pycsw using the provided docker/entrypoint.py. If not set, it will use 2 workers as Default. -**[manager]** +**profiles** + +- list of profiles to load at runtime (default is none). See :ref:`profiles` + +**manager** - **transactions**: whether to enable transactions (``true`` or ``false``). Default is ``false`` (see :ref:`transactions`) - **allowed_ips**: comma delimited list of IP addresses (e.g. 192.168.0.103), wildcards (e.g. 192.168.0.*) or CIDR notations (e.g. 192.168.100.0/24) allowed to perform transactions (see :ref:`transactions`) - **csw_harvest_pagesize**: when harvesting other CSW servers, the number of records per request to page by (default is 10) -**[metadata:main]** - -- **identification_title**: the title of the service -- **identification_abstract**: some descriptive text about the service -- **identification_keywords**: comma delimited list of keywords about the service -- **identification_keywords_type**: keyword type as per the `ISO 19115 MD_KeywordTypeCode codelist `_). Accepted values are ``discipline``, ``temporal``, ``place``, ``theme``, ``stratum`` -- **identification_fees**: fees associated with the service -- **identification_accessconstraints**: access constraints associated with the service -- **provider_name**: the name of the service provider -- **provider_url**: the URL of the service provider -- **contact_name**: the name of the provider contact -- **contact_position**: the position title of the provider contact -- **contact_address**: the address of the provider contact -- **contact_city**: the city of the provider contact -- **contact_stateorprovince**: the province or territory of the provider contact -- **contact_postalcode**: the postal code of the provider contact -- **contact_country**: the country of the provider contact -- **contact_phone**: the phone number of the provider contact -- **contact_fax**: the facsimile number of the provider contact -- **contact_email**: the email address of the provider contact -- **contact_url**: the URL to more information about the provider contact -- **contact_hours**: the hours of service to contact the provider -- **contact_instructions**: the how to contact the provider contact -- **contact_role**: the role of the provider contact as per the `ISO 19115 CI_RoleCode codelist `_). Accepted values are ``author``, ``processor``, ``publisher``, ``custodian``, ``pointOfContact``, ``distributor``, ``user``, ``resourceProvider``, ``originator``, ``owner``, ``principalInvestigator`` - -**[repository]** +**metadata** + +**metadata.identification** + +- **title**: the title of the service +- **description**: some descriptive text about the service +- **keywords**: list of keywords about the service +- **keywords_type**: keyword type as per the `ISO 19115 MD_KeywordTypeCode codelist `_). Accepted values are ``discipline``, ``temporal``, ``place``, ``theme``, ``stratum`` +- **fees**: fees associated with the service +- **accessconstraints**: access constraints associated with the service + +**metadata.provider** + +- **name**: the name of the service provider +- **url**: the URL of the service provider + +**metadata.contact** + +- **name**: the name of the provider contact +- **position**: the position title of the provider contact +- **address**: the address of the provider contact +- **city**: the city of the provider contact +- **stateorprovince**: the province or territory of the provider contact +- **postalcode**: the postal code of the provider contact +- **country**: the country of the provider contact +- **phone**: the phone number of the provider contact +- **fax**: the facsimile number of the provider contact +- **email**: the email address of the provider contact +- **url**: the URL to more information about the provider contact +- **hours**: the hours of service to contact the provider +- **instructions**: the how to contact the provider contact +- **role**: the role of the provider contact as per the `ISO 19115 CI_RoleCode codelist `_). Accepted values are ``author``, ``processor``, ``publisher``, ``custodian``, ``pointOfContact``, ``distributor``, ``user``, ``resourceProvider``, ``originator``, ``owner``, ``principalInvestigator`` + +**repository** - **database**: the full file path to the metadata database, in database URL format (see https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls) - **table**: the table name for metadata records (default is ``records``). If you are using PostgreSQL with a DB schema other than ``public``, qualify the table like ``myschema.table`` @@ -79,10 +90,10 @@ pycsw's runtime configuration is defined by ``default.cfg``. pycsw ships with a MaxRecords Handling ------------------- -The The following describes how ``maxRecords`` is handled by the configuration when handling OARec items or CSW ``GetRecords`` requests: +The The following describes how ``maxRecords`` is handled by the configuration when handling OGC API - Records items or CSW ``GetRecords`` requests: .. csv-table:: - :header: server.maxrecords,OARec limit/CSW GetRecords.maxRecords,Result + :header: server.maxrecords,OGC API - Records limit/CSW GetRecords.maxRecords,Result none set,none passed,10 (CSW default) 20,14,20 @@ -100,17 +111,17 @@ for deploying into `12 factor `_ environments for example Below is an example of how to integrate system environment variables in pycsw: -.. code-block:: ini +.. code-block:: yaml - [repository] - database=${PYCSW_REPOSITORY_DATABASE_URI} - table=${MY_TABLE} + repository: + database: ${PYCSW_REPOSITORY_DATABASE_URI} + table: ${MY_TABLE} Alternate Configurations ------------------------ -By default, pycsw loads ``default.cfg`` at runtime. To load an alternate configuration, modify ``csw.py`` to point to the desired configuration. Alternatively, pycsw supports explicitly specifiying a configuration by appending ``config=/path/to/default.cfg`` to the base URL of the service (e.g. ``http://localhost/pycsw/csw.py?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetCapabilities``). When the ``config`` parameter is passed by a CSW client, pycsw will override the default configuration location and subsequent settings with those of the specified configuration. +By default, pycsw loads ``default.yml`` at runtime. To load an alternate configuration, modify ``csw.py`` to point to the desired configuration. Alternatively, pycsw supports explicitly specifiying a configuration by appending ``config=/path/to/default.yml`` to the base URL of the service (e.g. ``http://localhost/pycsw/csw.py?config=tests/suites/default/default.yml&service=CSW&version=2.0.2&request=GetCapabilities``). When the ``config`` parameter is passed by a CSW client, pycsw will override the default configuration location and subsequent settings with those of the specified configuration. This also provides the functionality to deploy numerous CSW servers with a single pycsw installation. @@ -130,12 +141,12 @@ pycsw supports the following environment variables: Configuration file location ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -One option is using Apache's ``Alias`` and ``SetEnvIf`` directives. For example, given the base URL ``http://localhost/pycsw/csw.py?config=foo.cfg``, set the following in your Apache configuration: +One option is using Apache's ``Alias`` and ``SetEnvIf`` directives. For example, given the base URL ``http://localhost/pycsw/csw.py?config=foo.yml``, set the following in your Apache configuration: .. code-block:: none Alias /pycsw/csw-foo.py /var/www/pycsw/csw.py - SetEnvIf Request_URI "/pycsw/csw-foo.py" PYCSW_CONFIG=/var/www/pycsw/csw-foo.cfg + SetEnvIf Request_URI "/pycsw/csw-foo.py" PYCSW_CONFIG=/var/www/pycsw/csw-foo.yml. .. note:: @@ -152,10 +163,10 @@ Another option is to write a simple wrapper (e.g. ``csw-foo.sh``), which provide #!/bin/sh - export PYCSW_CONFIG=/var/www/pycsw/csw-foo.cfg + export PYCSW_CONFIG=/var/www/pycsw/csw-foo.yml /var/www/pycsw/csw.py -.. _`sample configuration`: https://github.com/geopython/pycsw/blob/master/default-sample.cfg +.. _`sample configuration`: https://github.com/geopython/pycsw/blob/master/default-sample.yml diff --git a/docs/distributedsearching.rst b/docs/distributedsearching.rst index 01b5fef9f..4c671cb02 100644 --- a/docs/distributedsearching.rst +++ b/docs/distributedsearching.rst @@ -15,7 +15,7 @@ Distributed Searching CSW 2 / 3 --------- -pycsw has the ability to perform distributed searching against other CSW servers. Distributed searching is disabled by default; to enable, ``server.federatedcatalogues`` must be set. A CSW client must issue a GetRecords request with ``csw:DistributedSearch`` specified, along with an optional ``hopCount`` attribute (see subclause 10.8.4.13 of the CSW specification). When enabled, pycsw will search all specified catalogues and return a unified set of search results to the client. Due to the distributed nature of this functionality, requests will take extra time to process compared to queries against the local repository. +pycsw has the ability to perform distributed searching against other CSW servers. Distributed searching is disabled by default; to enable, ``federatedcatalogues`` must be set. A CSW client must issue a GetRecords request with ``csw:DistributedSearch`` specified, along with an optional ``hopCount`` attribute (see subclause 10.8.4.13 of the CSW specification). When enabled, pycsw will search all specified catalogues and return a unified set of search results to the client. Due to the distributed nature of this functionality, requests will take extra time to process compared to queries against the local repository. Scenario: Federated Search ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -24,17 +24,17 @@ pycsw deployment with 3 configurations (CSW-1, CSW-2, CSW-3), subsequently provi pycsw realizes this functionality by supporting :ref:`alternate configurations `, and exposes the additional CSW endpoint(s) with the following design pattern: -CSW-1: ``http://localhost/pycsw/csw.py?config=CSW-1.cfg`` +CSW-1: ``http://localhost/pycsw/csw.py?config=CSW-1.yml`` -CSW-2: ``http://localhost/pycsw/csw.py?config=CSW-2.cfg`` +CSW-2: ``http://localhost/pycsw/csw.py?config=CSW-2.yml`` -CSW-3: ``http://localhost/pycsw/csw.py?config=CSW-3.cfg`` +CSW-3: ``http://localhost/pycsw/csw.py?config=CSW-3.yml`` -...where the ``*.cfg`` configuration files are configured for each respective metadata repository. The above CSW endpoints can be interacted with as usual. +...where the ``*.yml`` configuration files are configured for each respective metadata repository. The above CSW endpoints can be interacted with as usual. To federate the discovery of the three (3) portals into a unified search, pycsw realizes this functionality by deploying an additional configuration which acts as the superset of CSW-1, CSW-2, CSW-3: -CSW-all: ``http://localhost/pycsw/csw.py?config=CSW-all.cfg`` +CSW-all: ``http://localhost/pycsw/csw.py?config=CSW-all.yml`` This allows the client to invoke one (1) CSW GetRecords request, in which the CSW endpoint spawns the same GetRecords request to 1..n distributed CSW endpoints. Distributed CSW endpoints are advertised in CSW Capabilities XML via ``ows:Constraint``: @@ -43,9 +43,9 @@ This allows the client to invoke one (1) CSW GetRecords request, in which the CS ... - http://localhost/pycsw/csw.py?config=CSW-1.cfg - http://localhost/pycsw/csw.py?config=CSW-2.cfg - http://localhost/pycsw/csw.py?config=CSW-3.cfg + http://localhost/pycsw/csw.py?config=CSW-1.yml + http://localhost/pycsw/csw.py?config=CSW-2.yml + http://localhost/pycsw/csw.py?config=CSW-3.yml ... @@ -54,11 +54,12 @@ This allows the client to invoke one (1) CSW GetRecords request, in which the CS in the CSW-all configuration: -.. code-block:: none +.. code-block:: yaml - [server] - ... - federatedcatalogues=http://localhost/pycsw/csw.py?config=CSW-1.cfg,http://localhost/pycsw/csw.py?config=CSW-2.cfg,http://localhost/pycsw/csw.py?config=CSW-3.cfg + federatedcatalogues: + - http://localhost/pycsw/csw.py?config=CSW-1.yml + - http://localhost/pycsw/csw.py?config=CSW-2.yml + - http://localhost/pycsw/csw.py?config=CSW-3.yml At which point a CSW client request to CSW-all with ``distributedsearch=TRUE``, while specifying an optional ``hopCount``. Query network topology: @@ -93,12 +94,12 @@ Experimental support for distibuted searching is available in pycsw's OGC API - The ``federatedcatalogues`` directives must point to an OGC API - Records **collections** endpoint. -.. code-block:: none - - [server] - ... - federatedcatalogues=https://example.org/collections/collection1,https://example.org/collections/collection2 +.. code-block:: yaml + federatedcatalogues: + - https://example.org/collections/collection1 + - https://example.org/collections/collection2 + With the above configured, a distributed search can be invoked as follows: http://localhost/collections/metadata:main/items?distributed=true diff --git a/docs/docker.rst b/docs/docker.rst index aa5eaa9a3..d02de9881 100644 --- a/docs/docker.rst +++ b/docs/docker.rst @@ -69,24 +69,24 @@ pycsw configuration It is possible to supply a custom configuration file for pycsw as a bind mount or as a docker secret (in the case of docker swarm). The configuration file is searched at the value of the ``PYCSW_CONFIG`` environmental variable, -which defaults to ``/etc/pycsw/pycsw.cfg``. +which defaults to ``/etc/pycsw/pycsw.yml``. Supplying the configuration file via bind mount:: docker run \ --name pycsw \ --detach \ - --volume :/etc/pycsw/pycsw.cfg \ + --volume :/etc/pycsw/pycsw.yml \ --publish 8000:8000 \ geopython/pycsw Supplying the configuration file via docker secrets:: # first create a docker secret with the pycsw config file - docker secret create pycsw-config + docker secret create pycsw-config docker service create \ --name pycsw \ - --secret src=pycsw-config,target=/etc/pycsw/pycsw.cfg \ + --secret src=pycsw-config,target=/etc/pycsw/pycsw.yml \ --publish 8000:8000 geopython/pycsw @@ -112,7 +112,7 @@ PostgreSQL repositories ^^^^^^^^^^^^^^^^^^^^^^^ Specifying a PostgreSQL repository is just a matter of configuring a custom -pycsw.cfg file with the correct specification. +pycsw.yml file with the correct specification. Check `pycsw's github repository`_ for an example of a docker-compose/stack file that spins up a postgis database together with a pycsw instance. diff --git a/docs/hhypermap.rst b/docs/hhypermap.rst index b101d1f68..b66779fbe 100644 --- a/docs/hhypermap.rst +++ b/docs/hhypermap.rst @@ -14,6 +14,6 @@ HHypermap Setup pycsw is enabled and configured by default in HHypermap, so there are no additional steps required once HHypermap is setup. See the ``REGISTRY_PYCSW`` `hypermap/settings.py entries`_ for customizing pycsw within HHypermap. -The HHypermap plugin is managed outside of pycsw within the HHypermap project. HHypermap settings must ensure that ``REGISTRY_PYCSW['repository']['source']`` is set to``hypermap.search.pycsw_repository``. +The HHypermap plugin is managed outside of pycsw within the HHypermap project. HHypermap settings must ensure that ``REGISTRY_PYCSW['repository']['source']`` is set to ``hypermap.search.pycsw_repository``. .. _`hypermap/settings.py entries`: https://github.com/cga-harvard/Hypermap-Registry/blob/master/hypermap/settings.py diff --git a/docs/installation.rst b/docs/installation.rst index 8f66e4ef1..b4f7fdaa5 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -13,6 +13,7 @@ pycsw requires the following Python supporting libraries: - `lxml`_ for XML support - `SQLAlchemy`_ for database bindings - `pyproj`_ for coordinate transformations +- `PyYAML`_ for configuration management - `Shapely`_ for spatial query / geometry support - `OWSLib`_ for CSW client and metadata parser - `xmltodict`_ for working with XML similar to working with JSON @@ -23,9 +24,8 @@ OGC API - Records OGC API - Records support additionally requires the following: -- `Flask`_ for pycsw's default OARec endpoint +- `Flask`_ for pycsw's default OGC API - Records endpoint - `pygeofilter`_ for CQL parsing -- `PyYAML`_ for OpenAPI document handling .. note:: @@ -50,11 +50,11 @@ The 4 minute install: virtualenv pycsw && cd pycsw && . bin/activate git clone https://github.com/geopython/pycsw.git && cd pycsw pip3 install -e . && pip3 install -r requirements-standalone.txt - cp default-sample.cfg default.cfg - vi default.cfg + cp default-sample.yml default.yml + vi default.yml # adjust paths in - # - server.home - # - repository.database + # server.home + # repository.database # set server.url to http://localhost:8000/ # start server - CSW 2/3, OAI-PMH, OpenSearch, SRU (all endpoints at /) python3 pycsw/wsgi.py @@ -65,8 +65,8 @@ To enable OGC API - Records as well as the abovementioned search standards: .. code-block:: bash # configure which config file to use - export PYCSW_CONFIG=default.cfg - # start server - OARec and all services (various endpoints below) + export PYCSW_CONFIG=default.yml + # start server - OGC API - Records and all services (various endpoints below) python3 pycsw/wsgi_flask.py # OGC API - Records curl http://localhost:8000 @@ -200,14 +200,14 @@ For Windows installs, change the first line of ``csw.py`` to: Security -------- -By default, ``default.cfg`` is at the root of the pycsw install. If pycsw is setup outside an HTTP server's ``cgi-bin`` area, this file could be read. The following options protect the configuration: +By default, ``default.yml`` is at the root of the pycsw install. If pycsw is setup outside an HTTP server's ``cgi-bin`` area, this file could be read. The following options protect the configuration: -- move ``default.cfg`` to a non HTTP accessible area, and modify ``csw.py`` to point to the updated location +- move ``default.yml`` to a non HTTP accessible area, and modify ``csw.py`` to point to the updated location - configure web server to deny access to the configuration. For example, in Apache, add the following to ``httpd.conf``: .. code-block:: none - + order allow,deny deny from all diff --git a/docs/introduction.rst b/docs/introduction.rst index 9321d5321..b931bcb65 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -3,7 +3,7 @@ Introduction ============ -pycsw is an OARec and OGC CSW server implementation written in Python. +pycsw is an OGC API - Records and OGC CSW server implementation written in Python. Features ======== @@ -22,7 +22,7 @@ Features - implements Open Archives Initiative Protocol for Metadata Harvesting - supports ISO, Dublin Core, DIF, FGDC, Atom, GM03 and DataCite metadata models - CGI or WSGI deployment -- simple configuration +- simple YAML configuration - transactional capabilities (OGC API - Records and CSW-T) - flexible repository configuration - `GeoNode`_ connectivity diff --git a/docs/json.rst b/docs/json.rst index 74cd51130..6ed65fa3f 100644 --- a/docs/json.rst +++ b/docs/json.rst @@ -3,10 +3,10 @@ JSON Support ============ -OARec ------ +OGC API - Records +----------------- -pycsw fully supports the OARec JSON conformance class, which is the default +pycsw fully supports the OGC API - Records JSON conformance class, which is the default representation provided. CSW diff --git a/docs/migration-guide.rst b/docs/migration-guide.rst index 6f417b770..2d0909d54 100644 --- a/docs/migration-guide.rst +++ b/docs/migration-guide.rst @@ -9,9 +9,10 @@ over time to help with pycsw change management. pycsw 2.x to 3.0 Migration -------------------------- +- the default configuration is now in YAML format. See :ref:`configuration` for more information. A helper script (``pycsw-admin.py migrate-config``) is included for updating from the previous configuration format - the default endpoint for standalone deployments is now powered by ``pycsw/wsgi_flask.py`` (based on Flask) which supports ALL pycsw supported APIs. Make sure to use ``requirements-standalone.txt`` on top of ``requirements.txt`` to install Flask along with other standalone requirements - the previously used ``pycsw/wsgi.py`` can still be used for CSW only deployments or for applications that need to integrate pycsw as a library (e.g. Django applications). PyPI installations still use ``requirements.txt`` which does not install Flask by default -- the default endpoint ``/`` is now OARec +- the default endpoint ``/`` is now OGC API - Records - the CSW endpoint is now ``/csw`` - the OAI-PMH endpoint is now ``/oaipmh`` - the OpenSearch endpoint is now ``/opensearch`` diff --git a/docs/oaipmh.rst b/docs/oaipmh.rst index 61bb40757..d82c49021 100644 --- a/docs/oaipmh.rst +++ b/docs/oaipmh.rst @@ -8,8 +8,8 @@ pycsw supports the `The Open Archives Initiative Protocol for Metadata Harvestin OAI-PMH OpenSearch support is enabled by default. There are two ways to access OAI-PMH depending on the deployment pattern chosen. -OARec deployment ----------------- +OGC API - Records deployment +---------------------------- .. code-block:: bash diff --git a/docs/oarec-support.rst b/docs/oarec-support.rst index d09fbc89c..d4a35c89a 100644 --- a/docs/oarec-support.rst +++ b/docs/oarec-support.rst @@ -6,12 +6,12 @@ OGC API - Records Support Versions -------- -pycsw supports `OGC API - Records - Part 1: Core, version 1.0`_ (OARec) by default. +pycsw supports `OGC API - Records - Part 1: Core, version 1.0`_ by default. Request Examples ---------------- -As the OGC successor to CSW, OARec is a change in paradigm rooted in lowering +As the OGC successor to CSW, OGC API - Records is a change in paradigm rooted in lowering the barrier to entry, being webby/of the web, and focusing on developer experience/adoption. JSON and HTML output formats are both supported via the ``f`` parameter. diff --git a/docs/opensearch.rst b/docs/opensearch.rst index d7df9db3c..cafdc535c 100644 --- a/docs/opensearch.rst +++ b/docs/opensearch.rst @@ -6,8 +6,8 @@ OpenSearch Support pycsw OpenSearch support is enabled by default. There are two ways to access OpenSearch depending on the deployment pattern chosen. -OARec deployment ----------------- +OGC API - Records deployment +---------------------------- .. code-block:: bash diff --git a/docs/profiles.rst b/docs/profiles.rst index 8993969a5..68cef8d2d 100644 --- a/docs/profiles.rst +++ b/docs/profiles.rst @@ -55,7 +55,7 @@ Your profile plugin class (``FooProfile``) must implement all methods as per ``p Enabling Profiles ----------------- -All profiles are disabled by default. To specify profiles at runtime, set the ``server.profiles`` value in the :ref:`configuration` to the name of the package (in the ``pycsw/plugins/profiles`` directory). To enable multiple profiles, specify as a comma separated value (see :ref:`configuration`). +All profiles are disabled by default. To specify profiles at runtime, set the ``profiles`` value in the :ref:`configuration` to the name of the package (in the ``pycsw/plugins/profiles`` directory). To enable multiple profiles, specify as a comma separated value (see :ref:`configuration`). Testing ------- diff --git a/docs/repofilters.rst b/docs/repofilters.rst index 7f01a2fc0..75991816e 100644 --- a/docs/repofilters.rst +++ b/docs/repofilters.rst @@ -43,11 +43,11 @@ A default pycsw instance (with no ``repository.filters`` option) will always pro Suppose you wanted to deploy another pycsw instance which serves metadata from the same database, but only from a specific subset. Here we set the ``repository.filter`` option: -.. code-block:: text +.. code-block:: yaml - [repository] - database=sqlite:///records.db - filter=pycsw:ParentIdentifier = '33' + repository: + database: sqlite:///records.db + filter: pycsw:ParentIdentifier = '33' The same CSW `GetRecords` filter as per above then yields the following results: @@ -60,9 +60,9 @@ Another example: .. code-block:: text - [repository] - database=sqlite:///records.db - filter=pycsw:ParentIdentifier != '33' + repository:0 + database: sqlite:///records.db + filter: "pycsw:ParentIdentifier != '33'" The same CSW `GetRecords` filter as per above then yields the following results: diff --git a/docs/repositories.rst b/docs/repositories.rst index 23fcfdec2..32a0ccddd 100644 --- a/docs/repositories.rst +++ b/docs/repositories.rst @@ -6,7 +6,7 @@ Repository Plugins Overview -------- -pycsw allows for the implementation of custom repositories in order to connect to a backend different from the pycsw's default. This is especially useful when downstream applications manage their own metadata model/database/document store and want pycsw to connect to it directly instead of using pycsw's default model, thus creating duplicate repositories which then require syncronization/accounting. Repository plugins enable a single metadata backend which is independent from the pycsw setup. pycsw thereby becomes a pure wrapper around a given backend in providing OARec, CSW and other APIs atop a given application. +pycsw allows for the implementation of custom repositories in order to connect to a backend different from the pycsw's default. This is especially useful when downstream applications manage their own metadata model/database/document store and want pycsw to connect to it directly instead of using pycsw's default model, thus creating duplicate repositories which then require syncronization/accounting. Repository plugins enable a single metadata backend which is independent from the pycsw setup. pycsw thereby becomes a pure wrapper around a given backend in providing OGC API - Records, CSW and other APIs atop a given application. All outputschemas must be placed in the ``pycsw/plugins/outputschemas`` directory. @@ -26,7 +26,7 @@ Configuration - set pycsw's ``repository.source`` setting to the class which implements the custom repository: -.. code-block:: none +.. code-block:: yaml - [repository] - mappings='path.to.repo_plugin.MyRepository' + repository: + mappings: 'path.to.repo_plugin.MyRepository' diff --git a/docs/sitemaps.rst b/docs/sitemaps.rst index 2b185e20f..28f46dd29 100644 --- a/docs/sitemaps.rst +++ b/docs/sitemaps.rst @@ -7,7 +7,7 @@ XML Sitemaps .. code-block:: bash - pycsw-admin.py gen-sitemap --config default.cfg --output sitemap.xml + pycsw-admin.py gen-sitemap --config default.yml --output sitemap.xml The ``sitemap.xml`` file should be saved to an an area on your web server (parallel to or above your pycsw install location) to enable web crawlers to index your repository. diff --git a/docs/sru.rst b/docs/sru.rst index 3101ace0b..de956c5b3 100644 --- a/docs/sru.rst +++ b/docs/sru.rst @@ -8,8 +8,8 @@ pycsw supports the `Search/Retrieval via URL`_ search protocol implementation as SRU support is enabled by default. There are two ways to access SRU depending on the deployment pattern chosen. -OARec deployment ----------------- +OGC API - Records deployment +---------------------------- .. code-block:: bash diff --git a/docs/tools.rst b/docs/tools.rst index 08b99c322..f477460fb 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -3,10 +3,10 @@ Cataloguing and Metadata Tools ============================== -OARec Clients and Servers -------------------------- +OGC API - Records Clients and Servers +------------------------------------- -https://github.com/opengeospatial/ogcapi-records/blob/master/implementations.md +- `OGC API - Records official implementations `_ CSW Clients ----------- diff --git a/docs/xslt.rst b/docs/xslt.rst index 74c3f3be9..d2d492c98 100644 --- a/docs/xslt.rst +++ b/docs/xslt.rst @@ -11,11 +11,13 @@ To specify a custom XSLT transformation, you must map to input and output outputschemas supported by pycsw, where the input outputschema must match the metadata as ingested and stored in the repository. -.. code-block:: ini +.. code-block:: yaml + + xslt: + - input_os: http://www.opengis.net/cat/csw/2.0.2 + output_os: http://www.isotc211.org/2005/gmd + transform: tests/functionaltests/suites/xslt/custom.xslt - # custom XSLT (section format: xslt:input_xml_schema,output_xml_schema) - [xslt:http://www.opengis.net/cat/csw/2.0.2,http://www.isotc211.org/2005/gmd] - xslt=/path/to/my-custom-iso.xslt The ``xslt`` directive must point to a valid XSLT document on disk. diff --git a/pycsw/core/admin.py b/pycsw/core/admin.py index e2d9de742..0e182b015 100644 --- a/pycsw/core/admin.py +++ b/pycsw/core/admin.py @@ -4,7 +4,7 @@ # Authors: Tom Kralidis # Angelos Tzotsos # -# Copyright (c) 2015 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2015 Angelos Tzotsos # # Permission is hereby granted, free of charge, to any person @@ -43,7 +43,8 @@ from pycsw.core import metadata, repository, util from pycsw.core.etree import etree from pycsw.core.etree import PARSER -from pycsw.core.util import parse_ini_config +from pycsw.core.util import parse_ini_config, str2bool +from pycsw.ogc.api.util import get_typed_value, yaml_dump, yaml_load LOGGER = logging.getLogger(__name__) @@ -398,13 +399,12 @@ def cli(): def cli_setup_repository(ctx, config, verbosity): """Create repository tables and indexes""" - cfg = parse_ini_config(config) + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) try: repository.setup(cfg['repository']['database'], table=cfg['repository'].get('table')) except Exception as err: - import traceback - print(traceback.format_exc()) msg = f'ERROR: Repository already exists: {err}' raise click.ClickException(msg) from err @@ -421,7 +421,10 @@ def cli_setup_repository(ctx, config, verbosity): @CLI_OPTION_YES def cli_load_records(ctx, config, path, recursive, yes, verbosity): """Load metadata records from directory or file into repository""" - cfg = parse_ini_config(config) + + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) + context = pconfig.StaticContext() load_records( @@ -441,7 +444,10 @@ def cli_load_records(ctx, config, path, recursive, yes, verbosity): @CLI_OPTION_YES_PROMPT def cli_delete_records(ctx, config, yes, verbosity): """Delete all records from repository""" - cfg = parse_ini_config(config) + + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) + context = pconfig.StaticContext() delete_records( @@ -461,7 +467,10 @@ def cli_delete_records(ctx, config, yes, verbosity): writable=True, file_okay=False)) def cli_export_records(ctx, config, path, verbosity): """Dump metadata records from repository into directory""" - cfg = parse_ini_config(config) + + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) + context = pconfig.StaticContext() export_records( @@ -478,7 +487,10 @@ def cli_export_records(ctx, config, path, verbosity): @CLI_OPTION_CONFIG def cli_rebuild_db_indexes(ctx, config, verbosity): """Rebuild repository database indexes""" - cfg = parse_ini_config(config) + + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) + context = pconfig.StaticContext() repo = repository.Repository(cfg['repository']['database'], context, table=cfg['repository'].get('table')) @@ -491,7 +503,10 @@ def cli_rebuild_db_indexes(ctx, config, verbosity): @CLI_OPTION_CONFIG def cli_optimize_db(ctx, config, verbosity): """Optimize repository database""" - cfg = parse_ini_config(config) + + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) + context = pconfig.StaticContext() repo = repository.Repository(cfg['repository']['database'], context, table=cfg['repository'].get('table')) @@ -505,7 +520,10 @@ def cli_optimize_db(ctx, config, verbosity): @click.option('--url', '-u', 'url', help='URL of harvest endpoint') def cli_refresh_harvested_records(ctx, config, verbosity, url): """Refresh / harvest non-local records in repository""" - cfg = parse_ini_config(config) + + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) + context = pconfig.StaticContext() refresh_harvested_records( @@ -526,7 +544,10 @@ def cli_refresh_harvested_records(ctx, config, verbosity, url): dir_okay=False)) def cli_gen_sitemap(ctx, config, output, verbosity): """Generate XML Sitemap""" - cfg = parse_ini_config(config) + + with open(config, encoding='utf8') as fh: + cfg = yaml_load(fh) + context = pconfig.StaticContext() gen_sitemap( @@ -579,6 +600,95 @@ def cli_get_sysprof(ctx): click.echo(get_sysprof()) +@click.command('migrate-config') +@cli_callbacks +@click.pass_context +@CLI_OPTION_CONFIG +def cli_migrate_config(ctx, config, verbosity): + """Migrate pycsw ini config to YAML""" + + dict_ = { + 'server': {}, + 'logging': {}, + 'manager': {}, + 'metadata': { + 'identification': {}, + 'provider': {}, + 'contact': {}, + 'inspire': {} + }, + 'profiles': [], + 'federatedcatalogues': [], + 'repository': {} + } + + cfg = parse_ini_config(config) + + for name, value in cfg.items('server'): + if name == 'loglevel': + dict_['logging']['level'] = value + elif name == 'logfile': + dict_['logging']['logfile'] = value + elif name == 'profiles': + dict_[name] = value.split(',') + elif name == 'federatedcatalogues': + dict_[name] = value.split(',') + else: + dict_['server'][name] = get_typed_value(value) + + for name, value in cfg.items('metadata:main'): + if name.startswith('identification'): + new_key = name.replace('identification_', '') + if new_key == 'keywords': + dict_['metadata']['identification'][new_key] = value.split(',') + elif new_key == 'abstract': + dict_['metadata']['identification']['description'] = value + else: + dict_['metadata']['identification'][new_key] = get_typed_value(value) + + if name.startswith('provider'): + new_key = name.replace('provider_', '') + dict_['metadata']['provider'][new_key] = get_typed_value(value) + + if name.startswith('contact'): + new_key = name.replace('contact_', '') + dict_['metadata']['contact'][new_key] = get_typed_value(value) + + for name, value in cfg.items('manager'): + if name == 'allowed_ips': + dict_['manager'][name] = value.split(',') + elif name == 'transactions': + dict_['manager'][name] = str2bool(value) + else: + dict_['manager'][name] = get_typed_value(value) + + for name, value in cfg.items('repository'): + if name == 'facets': + dict_['repository'][name] = value.split(',') + else: + dict_['repository'][name] = get_typed_value(value) + + for name, value in cfg.items('metadata:inspire'): + if name == 'languages_supported': + dict_['metadata']['inspire'][name] = value.split(',') + elif name == 'enabled': + dict_['metadata']['inspire'][name] = str2bool(value) + elif name == 'gemet_keywords': + dict_['metadata']['inspire'][name] = value.split(',') + elif name == 'temp_extent': + begin, end = value.split('/') + dict_['metadata']['inspire'][name] = { + 'begin': begin, + 'end': end + } + else: + dict_['metadata']['inspire'][name] = get_typed_value(value) + + yaml_file = config.replace('.cfg', '.yml') + click.echo(f'Writing to {yaml_file}') + yaml_dump(dict_, yaml_file) + + cli.add_command(cli_setup_repository) cli.add_command(cli_load_records) cli.add_command(cli_export_records) @@ -588,5 +698,6 @@ def cli_get_sysprof(ctx): cli.add_command(cli_refresh_harvested_records) cli.add_command(cli_gen_sitemap) cli.add_command(cli_post_xml) -cli.add_command(cli_get_sysprof) cli.add_command(cli_validate_xml) +cli.add_command(cli_get_sysprof) +cli.add_command(cli_migrate_config) diff --git a/pycsw/core/log.py b/pycsw/core/log.py index b3d529011..dc39a940e 100644 --- a/pycsw/core/log.py +++ b/pycsw/core/log.py @@ -3,7 +3,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2015 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -28,69 +28,44 @@ # # ================================================================= + import logging +import sys LOGGER = logging.getLogger(__name__) -MSG_FORMAT = '%(asctime)s] [%(levelname)s] file=%(pathname)s \ -line=%(lineno)s module=%(module)s function=%(funcName)s %(message)s' - -TIME_FORMAT = '%a, %d %b %Y %H:%M:%S' - -LOGLEVELS = { - 'CRITICAL': logging.CRITICAL, - 'ERROR': logging.ERROR, - 'WARNING': logging.WARNING, - 'INFO': logging.INFO, - 'DEBUG': logging.DEBUG, - 'NOTSET': logging.NOTSET, -} - - -def setup_logger(config=None): - """Initialize logging facility""" - if config is None: - return None - - # Do not proceed if logging has not been set up. - if not (config.has_option('server', 'loglevel') or - config.has_option('server', 'logfile')): - return None - logfile = None - loglevel = 'NOTSET' - logging_handlers = [] - to_stdout = False +def setup_logger(logging_config): + """ + Setup configuration - loglevel = config.get('server', 'loglevel', fallback=loglevel) + :param logging_config: logging specific configuration - if loglevel not in LOGLEVELS.keys(): - raise RuntimeError( - 'Invalid server configuration (server.loglevel).') + :returns: void (creates logging instance) + """ - if loglevel != 'NOTSET': - if config.has_option('server', 'logfile'): - logfile = config.get('server', 'logfile') - if logfile.strip() != '': - logging_handlers.append(logging.FileHandler(logfile)) - else: # stdout - to_stdout = True - else: # stdout - to_stdout = True + log_format = \ + '[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' + date_format = '%Y-%m-%dT%H:%M:%SZ' - if to_stdout: - logging_handlers.append(logging.StreamHandler()) + loglevels = { + 'CRITICAL': logging.CRITICAL, + 'ERROR': logging.ERROR, + 'WARNING': logging.WARNING, + 'INFO': logging.INFO, + 'DEBUG': logging.DEBUG, + 'NOTSET': logging.NOTSET, + } - # Setup logging globally (not only for the pycsw module) - # based on the parameters passed. - logging.basicConfig(level=LOGLEVELS[loglevel], - format=MSG_FORMAT, - datefmt=TIME_FORMAT, - handlers=logging_handlers) + loglevel = loglevels[logging_config['level']] - LOGGER.info('Logging initialized (level: %s).', loglevel) + if 'logfile' in logging_config: + logging.basicConfig(level=loglevel, datefmt=date_format, + format=log_format, + filename=logging_config['logfile']) + else: + logging.basicConfig(level=loglevel, datefmt=date_format, + format=log_format, stream=sys.stdout) - if loglevel == 'DEBUG': # turn on CGI debugging - LOGGER.info('CGI debugging enabled.') - import cgitb - cgitb.enable() + LOGGER.debug('Logging initialized') + return diff --git a/pycsw/core/repository.py b/pycsw/core/repository.py index 480d75d47..2d62c0f84 100644 --- a/pycsw/core/repository.py +++ b/pycsw/core/repository.py @@ -5,7 +5,7 @@ # Angelos Tzotsos # Ricardo Garcia Silva # -# Copyright (c) 2019 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2015 Angelos Tzotsos # Copyright (c) 2017 Ricardo Garcia Silva # @@ -723,7 +723,6 @@ def setup(database, table, create_sfsql_tables=True, postgis_geometry_column='wk LOGGER.debug('Directory already exists') dbase = create_engine(database) - schema_name, table_name = table.rpartition(".")[::2] mdata = MetaData(dbase, schema=schema_name or None) diff --git a/pycsw/oaipmh.py b/pycsw/oaipmh.py index deda5796a..8a407bba0 100644 --- a/pycsw/oaipmh.py +++ b/pycsw/oaipmh.py @@ -3,7 +3,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2015 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -226,10 +226,10 @@ def response(self, response, kvp, repository, server_url): verbnode = etree.SubElement(node, util.nspath_eval('oai:%s' % verb, self.namespaces)) if verb == 'Identify': - etree.SubElement(verbnode, util.nspath_eval('oai:repositoryName', self.namespaces)).text = self.config.get('metadata:main', 'identification_title') + etree.SubElement(verbnode, util.nspath_eval('oai:repositoryName', self.namespaces)).text = self.config['metadata']['identification']['title'] etree.SubElement(verbnode, util.nspath_eval('oai:baseURL', self.namespaces)).text = url etree.SubElement(verbnode, util.nspath_eval('oai:protocolVersion', self.namespaces)).text = '2.0' - etree.SubElement(verbnode, util.nspath_eval('oai:adminEmail', self.namespaces)).text = self.config.get('metadata:main', 'contact_email') + etree.SubElement(verbnode, util.nspath_eval('oai:adminEmail', self.namespaces)).text = self.config['metadata']['contact']['email'] etree.SubElement(verbnode, util.nspath_eval('oai:earliestDatestamp', self.namespaces)).text = repository.query_insert('min') etree.SubElement(verbnode, util.nspath_eval('oai:deletedRecord', self.namespaces)).text = 'no' etree.SubElement(verbnode, util.nspath_eval('oai:granularity', self.namespaces)).text = 'YYYY-MM-DDThh:mm:ssZ' diff --git a/pycsw/ogc/api/oapi.py b/pycsw/ogc/api/oapi.py index d2ad4b51d..37cb047d5 100644 --- a/pycsw/ogc/api/oapi.py +++ b/pycsw/ogc/api/oapi.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -246,18 +246,18 @@ def gen_oapi(config, oapi_filepath, mode='ogcapi-records'): LOGGER.debug('Adding server info') oapi['info'] = { 'contact': { - 'email': config.get('metadata:main', 'contact_email'), - 'name': config.get('metadata:main', 'contact_name'), - 'url': config.get('metadata:main', 'contact_url') + 'email': config['metadata']['contact']['email'], + 'name': config['metadata']['contact']['name'], + 'url': config['metadata']['contact']['url'] }, 'version': __version__, - 'title': config.get('metadata:main', 'identification_title'), - 'description': config.get('metadata:main', 'identification_abstract') + 'title': config['metadata']['identification']['title'], + 'description': config['metadata']['identification']['description'] } oapi['servers'] = [{ - 'url': config.get('server', 'url'), - 'description': config.get('metadata:main', 'identification_abstract') + 'url': config['server'].get('url'), + 'description': config['metadata']['identification']['description'] }] LOGGER.debug('Adding paths') @@ -429,7 +429,7 @@ def gen_oapi(config, oapi_filepath, mode='ogcapi-records'): } } - if config.get('manager', 'transactions') == 'true': + if config['manager'].get('transactions', False): LOGGER.debug('Transactions enabled; adding post') path['post'] = { @@ -497,7 +497,7 @@ def gen_oapi(config, oapi_filepath, mode='ogcapi-records'): } } - if config.get('manager', 'transactions') == 'true': + if config['manager'].get('transactions', False): LOGGER.debug('Transactions enabled; adding put/delete') path['put'] = { diff --git a/pycsw/ogc/api/records.py b/pycsw/ogc/api/records.py index cc9219dc9..83c3179ee 100644 --- a/pycsw/ogc/api/records.py +++ b/pycsw/ogc/api/records.py @@ -3,7 +3,7 @@ # Authors: Tom Kralidis # Angelos Tzotsos # -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2021 Angelos Tzotsos # # Permission is hereby granted, free of charge, to any person @@ -29,7 +29,6 @@ # # ================================================================= -from configparser import ConfigParser import json import logging import os @@ -80,11 +79,11 @@ class API: """API object""" - def __init__(self, config: ConfigParser): + def __init__(self, config: dict): """ constructor - :param config: ConfigParser pycsw configuration dict + :param config: pycsw configuration dict :returns: `pycsw.ogc.api.API` instance """ @@ -92,7 +91,7 @@ def __init__(self, config: ConfigParser): self.mode = 'ogcapi-records' self.config = config - log.setup_logger(self.config) + log.setup_logger(self.config.get('logging', {})) if self.config['server']['url'].startswith('${'): LOGGER.debug(f"Server URL is an environment variable: {self.config['server']['url']}") @@ -102,22 +101,20 @@ def __init__(self, config: ConfigParser): LOGGER.debug(f'Server URL: {url_}') self.config['server']['url'] = url_.rstrip('/') - self.facets = self.config['repository'].get('facets', 'type').split(',') + self.facets = self.config['repository'].get('facets', ['type']) self.context = StaticContext() - LOGGER.debug('Setting maxrecords') + LOGGER.debug('Setting limit') try: - self.maxrecords = int(self.config['server']['maxrecords']) + self.limit = int(self.config['server']['maxrecords']) except KeyError: - self.maxrecords = 10 - LOGGER.debug(f'maxrecords: {self.maxrecords}') + self.limit= 10 + LOGGER.debug(f'limit: {self.limit}') - repo_filter = None - if self.config.has_option('repository', 'filter'): - repo_filter = self.config.get('repository', 'filter') + repo_filter = self.config['repository'].get('filter') - custom_mappings_path = self.config.get('repository', 'mappings', fallback=None) + custom_mappings_path = self.config['repository'].get('mappings') if custom_mappings_path is not None: md_core_model = load_custom_repo_mappings(custom_mappings_path) if md_core_model is not None: @@ -131,9 +128,9 @@ def __init__(self, config: ConfigParser): try: LOGGER.info('Loading default repository') self.repository = repository.Repository( - self.config.get('repository', 'database'), + self.config['repository']['database'], self.context, - table=self.config.get('repository', 'table'), + table=self.config['repository']['table'], repo_filter=repo_filter ) LOGGER.debug(f'Repository loaded {self.repository.dbtype}') @@ -208,11 +205,11 @@ def landing_page(self, headers_, args): response = { 'id': 'pycsw-catalogue', 'links': [], - 'title': self.config['metadata:main']['identification_title'], + 'title': self.config['metadata']['identification']['title'], 'description': - self.config['metadata:main']['identification_abstract'], + self.config['metadata']['identification']['description'], 'keywords': - self.config['metadata:main']['identification_keywords'].split(',') + self.config['metadata']['identification']['keywords'] } LOGGER.debug('Creating links') @@ -481,9 +478,9 @@ def queryables(self, headers_, args, collection='metadata:main'): properties2[key] = value if collection == 'metadata:main': - title = self.config['metadata:main']['identification_title'] + title = self.config['metadata']['identification']['title'] else: - title = self.config['metadata:main']['identification_title'] + title = self.config['metadata']['identification']['title'] virtual_collection = self.repository.query_ids([collection])[0] title = virtual_collection.title @@ -707,10 +704,10 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'): msg = 'Limit must be a positive integer' LOGGER.exception(msg) return self.get_exception(400, headers_, 'InvalidParameterValue', msg) - if limit > self.maxrecords: - limit = self.maxrecords + if limit > self.limit: + limit = self.limit else: - limit = self.maxrecords + limit = self.limit offset = int(args.get('offset', 0)) @@ -731,8 +728,8 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'): distributed = str2bool(args.get('distributed', False)) - if distributed and 'federatedcatalogues' in self.config['server']: - for fc in self.config['server']['federatedcatalogues'].split(','): + if distributed: + for fc in self.config.get('federatedcatalogues', []): LOGGER.debug(f'Running distributed search against {fc}') fc_url, _, fc_collection = fc.rsplit('/', 2) try: @@ -810,7 +807,7 @@ def items(self, headers_, json_post_data, args, collection='metadata:main'): }) if headers_['Content-Type'] == 'text/html': - response['title'] = self.config['metadata:main']['identification_title'] + response['title'] = self.config['metadata']['identification']['title'] response['collection'] = collection template = 'items.html' @@ -845,8 +842,8 @@ def item(self, headers_, args, collection, item): except IndexError: distributed = str2bool(args.get('distributed', False)) - if distributed and 'federatedcatalogues' in self.config['server']: - for fc in self.config['server']['federatedcatalogues'].split(','): + if distributed: + for fc in self.config.get('federatedcatalogues', []): LOGGER.debug(f'Running distributed item search against {fc}') fc_url, _, fc_collection = fc.rsplit('/', 2) try: @@ -865,7 +862,7 @@ def item(self, headers_, args, collection, item): return headers_, 200, record.xml if headers_['Content-Type'] == 'text/html': - response['title'] = self.config['metadata:main']['identification_title'] + response['title'] = self.config['metadata']['identification']['title'] response['collection'] = collection if 'json' in headers_['Content-Type']: @@ -882,7 +879,7 @@ def manage_collection_item(self, headers_, action='create', item=None, data=None :returns: tuple of headers, status code, content """ - if self.config.get('manager', 'transactions') != 'true': + if not self.config['manager']['transactions']: return self.get_exception( 405, headers_, 'InvalidParameterValue', 'transactions not allowed') @@ -986,8 +983,8 @@ def get_collection_info(self, collection_name: str = 'metadata:main', if collection_name == 'metadata:main': id_ = collection_name - title = self.config['metadata:main']['identification_title'] - description = self.config['metadata:main']['identification_abstract'] + title = self.config['metadata']['identification']['title'] + description = self.config['metadata']['identification']['description'] else: id_ = collection_name title = collection_info.get('title') @@ -1021,10 +1018,10 @@ def get_collection_info(self, collection_name: str = 'metadata:main', } if collection_name == 'metadata:main': - if self.config['server'].get('federatedcatalogues') is not None: + if 'federatedcatalogues' in self.config: LOGGER.debug('Adding federated catalogues') collection_info['federatedCatalogues'] = [] - for fc in self.config['server']['federatedcatalogues'].split(','): + for fc in self.config.get('federatedcatalogues', []): collection_info['federatedCatalogues'].append({ 'type': 'OGC API - Records', 'url': fc diff --git a/pycsw/ogc/api/templates/_base.html b/pycsw/ogc/api/templates/_base.html index e07509ced..52a0b1c55 100644 --- a/pycsw/ogc/api/templates/_base.html +++ b/pycsw/ogc/api/templates/_base.html @@ -2,11 +2,11 @@ - {% block title %}{{ config['metadata:main']['identification_title'] }} -{% endblock %} + {% block title %}{{ config['metadata']['identification']['title'] }} -{% endblock %} - - + + {% for link in data['links'] %} @@ -26,7 +26,7 @@
pycsw website -

{{ config['metadata:main']['identification_title'] }}

+

{{ config['metadata']['identification']['title'] }}

@@ -51,7 +51,7 @@

{{ config['metadata:main']['identification_title'] }}

{% if links_found.json == 0 %} JSON {% endif %} - | Contact + | Contact diff --git a/pycsw/ogc/api/templates/landing_page.html b/pycsw/ogc/api/templates/landing_page.html index eda1a23b7..b94ce802c 100644 --- a/pycsw/ogc/api/templates/landing_page.html +++ b/pycsw/ogc/api/templates/landing_page.html @@ -4,7 +4,7 @@