Skip to content
This repository has been archived by the owner on Dec 9, 2024. It is now read-only.

Commit

Permalink
feat: create local rdf catalog for harvest
Browse files Browse the repository at this point in the history
  • Loading branch information
NilsOveTen committed Mar 24, 2024
1 parent b72bb49 commit 398708b
Show file tree
Hide file tree
Showing 7 changed files with 2,343 additions and 467 deletions.
12 changes: 12 additions & 0 deletions mypy.ini
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,15 @@ ignore_missing_imports = True

[mypy-yaml.*]
ignore_missing_imports = True

[mypy-requests.*]
ignore_missing_imports = True

[mypy-datacatalogtordf.*]
ignore_missing_imports = True

[mypy-oastodcat.*]
ignore_missing_imports = True

[mypy-rdflib.*]
ignore_missing_imports = True
12 changes: 6 additions & 6 deletions noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def install_with_constraints(session: Session, *args: str, **kwargs: Any) -> Non
session.install(f"--constraint={requirements.name}", *args, **kwargs)


@nox_poetry.session(python=["3.9", "3.7"])
@nox_poetry.session(python=["3.9"])
def tests(session: Session) -> None:
"""Run the test suite."""
args = session.posargs or ["--cov"]
Expand All @@ -46,7 +46,7 @@ def black(session: Session) -> None:
session.run("black", *args)


@nox_poetry.session(python=["3.9", "3.7"])
@nox_poetry.session(python=["3.9"])
def lint(session: Session) -> None:
"""Lint using flake8."""
args = session.posargs or locations
Expand All @@ -63,7 +63,7 @@ def lint(session: Session) -> None:
session.run("flake8", *args)


@nox_poetry.session(python=["3.9", "3.7"])
@nox_poetry.session(python=["3.9"])
def safety(session: Session) -> None:
"""Scan dependencies for insecure packages."""
with tempfile.NamedTemporaryFile() as requirements:
Expand All @@ -80,23 +80,23 @@ def safety(session: Session) -> None:
session.run("safety", "check", f"--file={requirements.name}", "--full-report")


@nox_poetry.session(python=["3.9", "3.7"])
@nox_poetry.session(python=["3.9"])
def mypy(session: Session) -> None:
"""Type-check using mypy."""
args = session.posargs or locations
session.install("mypy")
session.run("mypy", *args)


@nox_poetry.session(python="3.7")
@nox_poetry.session(python="3.9")
def pytype(session: Session) -> None:
"""Run the static type checker using pytype."""
args = session.posargs or ["--disable=import-error", *locations]
session.install("pytype")
session.run("pytype", *args)


@nox_poetry.session(python=["3.9", "3.7"])
@nox_poetry.session(python=["3.9"])
def coverage(session: Session) -> None:
"""Upload coverage data."""
session.install("coverage[toml]", "codecov")
Expand Down
1,097 changes: 639 additions & 458 deletions poetry.lock

Large diffs are not rendered by default.

9 changes: 6 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,21 @@ authors = ["Stig B. Dørmænen <[email protected]>"]
license = "Apache-2.0"

[tool.poetry.dependencies]
python = "^3.7"
PyYAML = "^5.4.1"
python = ">=3.9,<3.11"
PyYAML = "^6.0"
click = "^8.1.3"
jsonpickle = "^2.2.0"
rdflib = "^6.3.2"
oastodcat = "^2.0.2"
requests = "^2.31.0"

[tool.poetry.dev-dependencies]
pytest = "^7.2.0"
black = "^22.10.0"
flake8 = "^5.0.4"
flake8-black = "^0.3.3"
mypy = "^0.982"
pytype = {version = "^2022.10.13", python = "3.7"}
pytype = {version = "^2022.10.13", python = "3.9"}
flake8-annotations = "^2.9.1"
flake8-bandit = "^4.1.1"
flake8-bugbear = "^22.10.25"
Expand Down
1,295 changes: 1,295 additions & 0 deletions specs/rdf/dsop_catalog.ttl

Large diffs are not rendered by default.

338 changes: 338 additions & 0 deletions specs/rdf/dsop_catalog_test.ttl

Large diffs are not rendered by default.

47 changes: 47 additions & 0 deletions src/dsop_api_spesifikasjoner/generateSpecification.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
from typing import Any, List

import click
import datacatalogtordf
from oastodcat import OASDataService
from rdflib.graph import Graph, URIRef
from requests import get
import yaml

from . import __version__
Expand Down Expand Up @@ -39,6 +43,11 @@ def main(template: Any, input: Any, directory: Any) -> None:
prod_catalog_filename = os.path.join(directory, "dsop_catalog.json")
Path("test").mkdir(parents=True, exist_ok=True)
test_catalog_filename = os.path.join(directory, "test", "dsop_catalog_test.json")
Path("rdf").mkdir(parents=True, exist_ok=True)
turtle_prod_catalog_filename = os.path.join(directory, "rdf", "dsop_catalog.ttl")
turtle_test_catalog_filename = os.path.join(
directory, "rdf", "dsop_catalog_test.ttl"
)
prod_catalog = Catalog(production=True)
test_catalog = Catalog(production=False)
# skipping first row, which is headers:
Expand Down Expand Up @@ -75,6 +84,13 @@ def main(template: Any, input: Any, directory: Any) -> None:
_write_catalog_file(prod_catalog_filename, prod_catalog)
_write_catalog_file(test_catalog_filename, test_catalog)

_write_catalog_rdf_file(
turtle_prod_catalog_filename, create_catalog_graph(prod_catalog)
)
_write_catalog_rdf_file(
turtle_test_catalog_filename, create_catalog_graph(test_catalog)
)


def _write_spec_to_file(specification_filedirectory: str, spec: dict) -> None:
with open(specification_filedirectory, "w", encoding="utf-8") as outfile:
Expand Down Expand Up @@ -113,6 +129,11 @@ def _write_catalog_file(catalog_filename: str, catalog: Catalog) -> None:
)


def _write_catalog_rdf_file(catalog_filename: str, catalog: Graph) -> None:
with open(catalog_filename, "w", encoding="utf-8") as catalogfile:
catalogfile.write(catalog.serialize(format="turtle"))


def _generateSpec(template: dict, bank: List[str], production: bool) -> dict:
"""Generate spec based on template for bank."""
# Need to do a deepcopy to actually copy the template into a new object.
Expand All @@ -134,3 +155,29 @@ def _generateSpec(template: dict, bank: List[str], production: bool) -> dict:
server["description"] = "test"
specification["servers"].append(server)
return specification


def create_catalog_graph(catalog: Catalog) -> Graph:
"""Create a graph based on catalog and persist to store."""
# Use datacatalogtordf and oastodcat to create a graph and persist:
g = datacatalogtordf.Catalog()
g.identifier = URIRef(catalog.identifier)
g.title = catalog.title
g.description = catalog.description
g.publisher = catalog.publisher

for api in catalog.apis:
with get(api.url, timeout=5) as response:
if response.status_code == 200:
api_spec = response.text
oas = yaml.safe_load(api_spec)

oas_spec = OASDataService(api.url, oas, api.identifier)
oas_spec.conforms_to = api.conformsTo
oas_spec.publisher = api.publisher

# Add dataservices to catalog:
for dataservice in oas_spec.dataservices:
g.services.append(dataservice)

return g._to_graph()

0 comments on commit 398708b

Please sign in to comment.