Feature/add cli test

.github/workflows/unit-test.yml

@@ -0,0 +1,57 @@
+
+name: Unit tests
+
+on:
+  pull_request:
+    paths:
+    - ".github/workflows/unit-test.yml"
+    - "**/*.py"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+env:
+  POSTGRES_VERSION: "15"
+  POSTGIS_VERSION: "3.0"
+  POSTGRES_DB: ign
+  POSTGRES_USER: ign
+  POSTGRES_PASSWORD: ign
+  POSTGRES_HOST: "172.17.0.1"
+
+jobs:
+  test:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+
+    container: 'ghcr.io/ignf/route-graph-generator:develop'
+
+    services:
+      postgis:
+        image: postgis/postgis # unable to handle var in image name. Was: "postgis/postgis:${POSTGRES_VERSION}-${POSTGIS_VERSION}-alpine"
+        env:
+          POSTGRES_DB: ign # mandatory duplicate
+          POSTGRES_USER: ign # mandatory duplicate
+          POSTGRES_PASSWORD: ign # mandatory duplicate
+        ports:
+          # Maps tcp port 5555 on service container to the host
+          - 5555:5432
+
+    steps:
+
+      - uses: actions/checkout@v3
+
+      - name: Install test dependencies
+        run: |
+            pip install -r requirements/testing.txt
+
+      - name: Run test
+        run: |
+          python -m pytest
+
+      - name: Get Coverage
+        uses: orgoro/coverage@v3
+        with:
+          coverageFile: coverage.xml
+          token: ${{ secrets.GITHUB_TOKEN }}
+

docs/development/testing.md

@@ -0,0 +1,33 @@
+# Tests
+
+## Prérequis de l'environnement de test
+
+- Docker >= 20.10
+- Construction de l'image 
+
+La construction de l'image est lancée avec une commande :  
+
+```sh
+docker build -t r2gg-debian -f docker/debian/Dockerfile .
+```
+
+## Initialisation de la base de données de tests
+
+```sh
+```
+docker-compose -f tests/dev/docker-compose.dev.yml up -d
+```
+
+Mot de passe de l'utilisateur en base : `ign`
+
+## Exécution des tests dans l'image docker
+
+```sh
+docker run -it -v $(pwd):/r2gg r2gg-debian bash
+```
+
+```sh
+cd /r2gg
+pip install -U -r requirements/testing.txt
+pytest
+```

r2gg/_configure.py

@@ -33,7 +33,8 @@ def configure():
     """
     parser = argparse.ArgumentParser()
     parser.add_argument('config_file_path', type=str)
-    config_path = parser.parse_known_args()[0].config_file_path
+    args = parser.parse_args()
+    config_path = args.config_file_path
 
     # Récupération de l'objet 'génération' qui contient toute la config
     config = config_from_path(config_path)['generation']

requirements/testing.txt

@@ -0,0 +1,3 @@
+# Testing dependencies
+# --------------------
+pytest-cov>=4,<5

setup.cfg

@@ -0,0 +1,31 @@
+# -- Tests ----------------------------------------------
+[tool:pytest]
+addopts =
+    --junitxml=junit/test-results.xml
+    --cov-config=setup.cfg
+    --cov=r2gg
+    --cov-report=html
+    --cov-report=term
+    --cov-report=xml
+    --ignore=tests/_wip/
+junit_family = xunit2
+norecursedirs = .* build dev development dist docs CVS fixtures _darcs {arch} *.egg venv _wip
+python_files = test_*.py
+testpaths = tests
+
+[coverage:run]
+branch = True
+omit =
+    .venv/*
+    docs/*
+    *tests*
+
+[coverage:report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+
+ignore_errors = True
+show_missing = True

tests/config/bdtopo_base.json

@@ -0,0 +1 @@
+{"host": "172.17.0.1", "database": "ign", "user": "ign", "password": "ign", "port": "5555"}

tests/config/pivot_base.json

@@ -0,0 +1 @@
+{"host": "172.17.0.1", "database": "pivot", "user": "ign", "password": "ign", "port": "5555"}

tests/config/sql2pivot.json

@@ -0,0 +1,43 @@
+{
+  "generation": {
+    "general":{
+      "id": "sql2pivot",
+      "logs": {"configFile": "/home/docker/config/sample_log_config.json"},
+      "overwrite": true,
+      "operation": "creation"
+    },
+    "bases": [
+      {
+        "id": "bdtopo",
+        "type": "bdd",
+        "configFile": "{src_dir}/tests/config/bdtopo_base.json",
+        "schema": "input"
+      },
+      {
+        "id": "pivot",
+        "type": "bdd",
+        "configFile": "{src_dir}/tests/config/pivot_base.json",
+        "schema": "output"
+      }
+    ],
+    "workingSpace": {
+      "directory": "/home/docker/data/generation/",
+      "baseId": "pivot"
+    },
+    "resource": {
+      "id": "bdtopo_conversion",
+      "sources": [
+        {
+          "id": "output_source",
+          "type": "pgr",
+          "bbox": "-180,-90,180, 90",
+          "mapping":
+          {"source": {"baseId": "bdtopo"},
+            "conversion":
+            {"file": "{src_dir}/sql/bdtopo_v3.3.sql"}
+          }
+        }
+      ]
+    }
+  }
+}

tests/dev/docker-compose.dev.yml

@@ -0,0 +1,11 @@
+version: "3"
+services:
+  r2gg-db-postgis:
+    image: postgis/postgis:15-3.3-alpine
+    environment:
+      POSTGRES_DB: ign
+      POSTGRES_PASSWORD: ign
+      POSTGRES_USER: ign
+      POSTGRES_INITDB_ARGS: "--data-checksums"
+    ports:
+      - 5555:5432

tests/test_cli.py

@@ -0,0 +1,105 @@
+import argparse
+import json
+import os
+from pathlib import Path
+from urllib import request
+
+# 3rd party
+import pytest
+from unittest.mock import patch
+
+from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+
+from r2gg import cli
+
+import psycopg2 as psycopg
+
+cur_dir = Path(__file__).parent.absolute()
+
+# ############################################################################
+# ########## Classes #############
+# ################################
+HOST = os.environ.get("POSTGRES_HOST", "172.17.0.1")
+PORT = os.environ.get("PORT", 5555)
+DBNAME = os.environ.get("POSTGRES_DB", "ign")
+USER = os.environ.get("POSTGRES_USER", "ign")
+PASS = os.environ.get("POSTGRES_PASSWORD", "ign")
+
+# Schemas are defined in input configuration .json files
+INPUT_SCHEMA = "input"
+OUTPUT_SCHEMA = "output"
+
+TRONCON_ROUTE_URL = "https://storage.gra.cloud.ovh.net/v1/AUTH_366279ce616242ebb14161b7991a8461/road2/troncon_route_marseille10.sql"
+NON_COMMUNICATION_URL = "https://storage.gra.cloud.ovh.net/v1/AUTH_366279ce616242ebb14161b7991a8461/road2/non_communication_marseille10.sql"
+
+@pytest.fixture
+def init_database(tmp_path) -> None:
+    """Init database for test."""
+
+    con = psycopg.connect(host=HOST, dbname=DBNAME, user=USER, password=PASS, port=PORT)
+    con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    con.cursor().execute(f"CREATE SCHEMA IF NOT EXISTS {INPUT_SCHEMA}")
+    con.cursor().execute(f"CREATE DATABASE pivot")
+    con.commit()
+    con.close()
+
+    # Insert test data
+    con = psycopg.connect(host=HOST, dbname=DBNAME, user=USER, password=PASS, port=PORT)
+    cur = con.cursor()
+    cur.execute(f"SET search_path TO {INPUT_SCHEMA}")
+
+    request.urlretrieve(TRONCON_ROUTE_URL, tmp_path / "troncon_route_marseille10.sql")
+    request.urlretrieve(NON_COMMUNICATION_URL, tmp_path / "non_communication_marseille10.sql")
+
+    with open(str(tmp_path / "troncon_route_marseille10.sql"), mode="r") as sql_script:
+        cur.execute(sql_script.read())
+    with open(str(tmp_path / "non_communication_marseille10.sql"), mode="r") as sql_script:
+        cur.execute(sql_script.read())
+
+    con.commit()
+    con.close()
+
+    # Add extensions to pivot
+    con = psycopg.connect(host=HOST, dbname="pivot", user=USER, password=PASS, port=PORT)
+    con.cursor().execute(f"CREATE SCHEMA IF NOT EXISTS {OUTPUT_SCHEMA}")
+    con.cursor().execute("CREATE EXTENSION IF NOT EXISTS postgres_fdw")
+    con.cursor().execute("CREATE EXTENSION IF NOT EXISTS Postgis")
+    con.commit()
+    con.close()
+    yield
+
+    con = psycopg.connect(host=HOST, dbname=DBNAME, user=USER, password=PASS, port=PORT)
+    con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    con.cursor().execute("DROP DATABASE pivot")
+    con.cursor().execute(f"DROP TABLE {INPUT_SCHEMA}.non_communication")
+    con.cursor().execute(f"DROP TABLE {INPUT_SCHEMA}.troncon_de_route")
+    con.commit()
+    con.close()
+
+
+def update_src_dir_json(file_path: Path, output_file_path: Path):
+    """Update .json file to replace {src_dir} by current source directory
+
+    Parameters
+    ----------
+    file_path (Path) database .json file path
+    """
+    with open(file_path, mode="r") as f:
+        content = f.read().replace("{src_dir}", str(Path(cur_dir / "..").resolve()))
+
+    with open(output_file_path, mode="w") as f:
+        f.write(content)
+
+
+def test_sql2pivot(init_database):
+    """Test simple run of cli for pivot base creation."""
+
+    # Update input json file to indicate current source directory
+    update_src_dir_json(cur_dir / "config" / "sql2pivot.json", cur_dir / "config" / "updated_sql2pivot.json")
+
+    # mock ArgumentParser for configuration file
+    with patch("argparse.ArgumentParser.parse_args") as parse_arg_mock:
+        parse_arg_mock.return_value = argparse.Namespace(
+            config_file_path=str(cur_dir / "config" / "updated_sql2pivot.json"))
+        # Run conversion
+        cli.sql2pivot()