refactor: project re-struct phase1, use hatch as build tool (#310)

This is the phase1 project re-struct, including the following changes:

1. use hatch instead of setuptools for CI & package build,
2. use /src/ project layout.
3. split ota_proxy out of otaclient package.

Other changes are related to the above changes, including:
1. fix up import paths accordingly.

.github/workflows/release.yml

@@ -20,12 +20,13 @@ jobs:
 
       - name: install build deps
         run: |
-          python -m pip install -U pip build
+          python -m pip install -U pip
+          python -m pip install -U hatch
           python -m pip install -r proto/requirements.txt
 
       - name: build otaclient package
         run: |
-          python -m build --wheel --outdir dist/ .
+          hatch build -t wheel
 
       - name: build a proto binary
         run: |

docker/docker-compose_tests.yml

@@ -8,11 +8,7 @@ services:
     container_name: ota-test
     environment:
       OUTPUT_DIR: /test_result
-      OTACLIENT_DIR: /ota-client
       CERTS_DIR: /certs
     volumes:
-      - ../pyproject.toml:/ota-client/pyproject.toml:ro
-      - ../.flake8:/ota-client/.flake8:ro
-      - ../otaclient:/ota-client/otaclient:ro
-      - ../tests:/ota-client/tests:ro
+      - ..:/otaclient_src:ro
       - ../test_result:/test_result:rw

docker/test_base/Dockerfile

@@ -4,11 +4,9 @@ ENV DEBIAN_FRONTEND=noninteractive
 ARG KERNEL_VERSION="5.8.0-53-generic"
 ARG BASE_IMG_URL="http://cdimage.ubuntu.com/ubuntu-base/releases/20.04/release/ubuntu-base-20.04.1-base-amd64.tar.gz"
 ARG OTA_METADATA_REPO="https://github.com/tier4/ota-metadata"
-ENV OTA_CLIENT_DIR="/ota-client"
 ENV OTA_IMAGE_SERVER_ROOT="/ota-image"
 ENV OTA_IMAGE_DIR="${OTA_IMAGE_SERVER_ROOT}/data"
 ENV CERTS_DIR="/certs"
-ENV VENV="${OTA_CLIENT_DIR}/.venv"
 ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement"
 
 
@@ -43,17 +41,10 @@ RUN set -eu; \
 # NOTE: include special identifiers #?; into the pathname
 RUN echo -n "${SPECIAL_FILE}" > "${OTA_IMAGE_DIR}/${SPECIAL_FILE}"
 
-# install otaclient dependencies
-COPY ./otaclient/requirements.txt /tmp/requirements.txt
-COPY ./tests/requirements.txt /tmp/requirements-dev.txt
+# install hatch
 RUN set -eu; \
-    python3 -m venv ${VENV}; \
-    source ${VENV}/bin/activate; \
     python3 -m pip install --no-cache-dir -q -U pip; \
-    python3 -m pip install \
-        --no-cache-dir -q \
-        -r /tmp/requirements.txt \
-        -r /tmp/requirements-dev.txt
+    python3 -m pip install --no-cache-dir -U hatch
 
 # generate test certs and sign key
 COPY --chmod=755 ./tests/keys/gen_certs.sh /tmp/certs/
@@ -98,5 +89,4 @@ RUN set -eu; \
 COPY ./docker/test_base/entry_point.sh /entry_point.sh
 RUN chmod +x /entry_point.sh
 
-WORKDIR ${OTA_CLIENT_DIR}
 ENTRYPOINT [ "/entry_point.sh" ]

docker/test_base/entry_point.sh

@@ -1,32 +1,24 @@
 #!/bin/bash
 set -eu
 
-OTA_CLIENT_DIR="${OTACLIENT_DIR:-/ota-client}"
+TEST_ROOT=/test_root
+OTACLIENT_SRC=/otaclient_src
 OUTPUT_DIR="${OUTPUT_DIR:-/test_result}"
 CERTS_DIR="${CERTS_DIR:-/certs}"
-VENV="${OTA_CLIENT_DIR}/.venv"
+
+# copy the source code as source is read-only
+cp -R "${OTACLIENT_SRC}" "${TEST_ROOT}"
 
 # copy the certs generated in the docker image to otaclient dir
 echo "setup certificates for testing..."
-cd ${OTA_CLIENT_DIR}
+cd ${TEST_ROOT}
 mkdir -p ./certs
 cp -av ${CERTS_DIR}/root.pem ./certs/1.root.pem
 cp -av ${CERTS_DIR}/interm.pem ./certs/1.interm.pem
 
-source ${VENV}/bin/activate
-
-# setup dependencies unconditionally
-# NOTE: if we only do minor modified to the requirements.txt,
-#       we don't have to rebuild the image, just call pip here
-APP_DEPENDENCIES="${OTA_CLIENT_DIR}/otaclient/requirements.txt"
-[ -f "$APP_DEPENDENCIES" ] && echo "setup app dependencies..." && \
-    python3 -m pip install --no-cache-dir -q -r $APP_DEPENDENCIES
-TESTS_DEPENDENCIES="${OTA_CLIENT_DIR}/tests/requirements.txt"
-[ -f "$TESTS_DEPENDENCIES" ] && echo "setup tests dependencies..." && \
-    python3 -m pip install --no-cache-dir -q -r $TESTS_DEPENDENCIES
-
 # exec the input params
 echo "execute test with coverage"
-cd ${OTA_CLIENT_DIR}
-coverage run -m pytest --junit-xml=${OUTPUT_DIR}/pytest.xml ${@:-}
-coverage xml -o ${OUTPUT_DIR}/coverage.xml
+cd ${TEST_ROOT}
+hatch env create dev
+hatch run dev:coverage run -m pytest --junit-xml=${OUTPUT_DIR}/pytest.xml ${@:-}
+hatch run dev:coverage xml -o ${OUTPUT_DIR}/coverage.xml

pyproject.toml

@@ -1,8 +1,8 @@
 [build-system]
-build-backend = "setuptools.build_meta"
+build-backend = "hatchling.build"
 requires = [
-  "setuptools>=61",
-  "setuptools_scm[toml]>=6.4",
+  "hatch-vcs",
+  "hatchling>=1.20",
 ]
 
 [project]
@@ -21,27 +21,54 @@ classifiers = [
   "Programming Language :: Python :: 3.12",
 ]
 dynamic = [
-  "dependencies",
-  "optional-dependencies",
   "version",
 ]
+dependencies = [
+  "aiofiles==22.1",
+  "aiohttp<3.10.0,>=3.9.5",
+  "cryptography<43.0.0,>=42.0.4",
+  "grpcio<1.54.0,>=1.53.2",
+  "protobuf<4.22.0,>=4.21.12",
+  "pydantic==2.7",
+  "pydantic-settings==2.2.1",
+  "pyOpenSSL==24.1",
+  "PyYAML>=3.12",
+  "requests<2.32.0,>=2.31",
+  'typing_extensions>=4.6.3; python_version < "3.11"',
+  "urllib3<2.0.0,>=1.26.8",
+  "uvicorn[standard]==0.20",
+  "zstandard==0.18",
+]
+[project.optional-dependencies]
+dev = [
+  "black",
+  "coverage",
+  "flake8",
+  "isort",
+  "pytest==7.1.2",
+  "pytest-asyncio==0.21",
+  "pytest-mock==3.8.2",
+  "requests-mock",
+]
 [project.urls]
 Source = "https://github.com/tier4/ota-client"
 
-[tool.setuptools.dynamic]
-dependencies = { file = ["otaclient/requirements.txt"] }
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.hooks.vcs]
+version-file = "src/_otaclient_version.py"
 
-[tool.setuptools.dynamic.optional-dependencies]
-test = { file = ["tests/requirements.txt"] }
+[tool.hatch.build.targets.sdist]
+exclude = ["/tools"]
 
-[tool.setuptools.packages.find]
-include = ["otaclient*"]
-namespaces = false
+[tool.hatch.build.targets.wheel]
+only-include = ["src"]
+sources = ["src"]
 
-[tool.setuptools_scm]
-write_to = "otaclient/_version.py"
-version_scheme = "post-release"
-local_scheme = "no-local-version"
+[tool.hatch.envs.dev]
+type = "virtual"
+features = ["dev"]
 
 [tool.black]
 line-length = 88

otaclient/.gitignore → src/.gitignore

@@ -1,2 +1,2 @@
 # generated version file by build
-_version.py
+_otaclient_version.py

otaclient/__init__.py → src/otaclient/__init__.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 try:
-    from otaclient._version import __version__, version
+    from _otaclient_version import __version__, version
 except ImportError:
     # unknown version
     version = __version__ = "0.0.0"

otaclient/_utils/typing.py → src/otaclient/_utils/typing.py

@@ -15,14 +15,15 @@
 
 from __future__ import annotations
 
+from enum import Enum
 from pathlib import Path
 from typing import Any, Callable, TypeVar, Union
 
 from pydantic import Field
 from typing_extensions import Annotated, ParamSpec
 
 P = ParamSpec("P")
-T = TypeVar("T")
+T = TypeVar("T", bound=Enum)
 RT = TypeVar("RT")
 
 StrOrPath = Union[str, Path]

otaclient/app/downloader.py → src/otaclient/app/downloader.py

@@ -48,8 +48,8 @@
 from urllib3.response import HTTPResponse
 from urllib3.util.retry import Retry
 
+from ota_proxy import OTAFileCacheControl
 from otaclient._utils import copy_callable_typehint
-from otaclient.ota_proxy import OTAFileCacheControl
 
 from .common import wait_with_backoff
 from .configs import config as cfg

otaclient/app/log_setting.py → src/otaclient/app/log_setting.py

@@ -100,5 +100,5 @@ def configure_logging():
         # NOTE: "otaclient" logger will be the root logger for all loggers name
         #       starts with "otaclient.", and the settings will affect its child loggers.
         #       For example, settings for "otaclient" logger will also be effective to
-        #       "otaclient.app.*" logger and "otaclient.ota_proxy.*" logger.
+        #       "otaclient.app.*" logger and "ota_proxy.*" logger.
         _otaclient_logger.addHandler(ch)

otaclient/app/ota_client_stub.py → src/otaclient/app/ota_client_stub.py

@@ -28,10 +28,10 @@
 
 from typing_extensions import Self
 
+from ota_proxy import OTAProxyContextProto
+from ota_proxy import config as local_otaproxy_cfg
+from ota_proxy import subprocess_otaproxy_launcher
 from otaclient.configs.ecu_info import ECUContact
-from otaclient.ota_proxy import OTAProxyContextProto
-from otaclient.ota_proxy import config as local_otaproxy_cfg
-from otaclient.ota_proxy import subprocess_otaproxy_launcher
 
 from . import log_setting
 from .boot_control._common import CMDHelperFuncs
@@ -65,7 +65,7 @@ def __init__(
         self._external_cache_dev_mp = external_cache_dev_mp
         self._external_cache_data_dir = external_cache_path
 
-        self.logger = logging.getLogger("otaclient.ota_proxy")
+        self.logger = logging.getLogger("ota_proxy")
 
     @property
     def extra_kwargs(self) -> Dict[str, Any]:
@@ -85,9 +85,9 @@ def _subprocess_init(self):
         # configure logging for otaproxy subprocess
         # NOTE: on otaproxy subprocess, we first set log level of the root logger
         #       to CRITICAL to filter out third_party libs' logging(requests, urllib3, etc.),
-        #       and then set the otaclient.ota_proxy logger to DEFAULT_LOG_LEVEL
+        #       and then set the ota_proxy logger to DEFAULT_LOG_LEVEL
         log_setting.configure_logging()
-        otaproxy_logger = logging.getLogger("otaclient.ota_proxy")
+        otaproxy_logger = logging.getLogger("ota_proxy")
         otaproxy_logger.setLevel(cfg.DEFAULT_LOG_LEVEL)
         self.logger = otaproxy_logger
 

otaclient/app/ota_metadata.py → src/otaclient/app/ota_metadata.py

@@ -71,7 +71,7 @@
 from OpenSSL import crypto
 from typing_extensions import Self
 
-from otaclient.ota_proxy import OTAFileCacheControl
+from ota_proxy import OTAFileCacheControl
 
 from .common import RetryTaskMap, get_backoff, urljoin_ensure_base
 from .configs import config as cfg

tests/conftest.py

@@ -42,7 +42,7 @@ class TestConfiguration:
     OTACLIENT_STUB_MODULE_PATH = "otaclient.app.ota_client_stub"
     OTACLIENT_SERVICE_MODULE_PATH = "otaclient.app.ota_client_service"
     OTAMETA_MODULE_PATH = "otaclient.app.ota_metadata"
-    OTAPROXY_MODULE_PATH = "otaclient.ota_proxy"
+    OTAPROXY_MODULE_PATH = "ota_proxy"
     CREATE_STANDBY_MODULE_PATH = "otaclient.app.create_standby"
     MAIN_MODULE_PATH = "otaclient.app.main"
 

tests/test_ota_client_stub.py

@@ -24,6 +24,8 @@
 import pytest
 from pytest_mock import MockerFixture
 
+from ota_proxy import OTAProxyContextProto
+from ota_proxy.config import Config as otaproxyConfig
 from otaclient.app.ota_client import OTAServicer
 from otaclient.app.ota_client_call import OtaClientCall
 from otaclient.app.ota_client_stub import (
@@ -34,8 +36,6 @@
 from otaclient.app.proto import wrapper
 from otaclient.configs.ecu_info import ECUInfo, parse_ecu_info
 from otaclient.configs.proxy_info import ProxyInfo, parse_proxy_info
-from otaclient.ota_proxy import OTAProxyContextProto
-from otaclient.ota_proxy.config import Config as otaproxyConfig
 from tests.conftest import cfg
 from tests.utils import compare_message
 

tests/test_ota_proxy/test_cache_control_header.py

@@ -17,7 +17,7 @@
 
 import pytest
 
-from otaclient.ota_proxy import OTAFileCacheControl
+from ota_proxy import OTAFileCacheControl
 
 
 @pytest.mark.parametrize(

tests/test_ota_proxy/test_cachedb.py

@@ -22,18 +22,18 @@
 
 import pytest
 
-from otaclient.ota_proxy import config as cfg
-from otaclient.ota_proxy.orm import NULL_TYPE
-from otaclient.ota_proxy.ota_cache import CacheMeta, OTACacheDB
-from otaclient.ota_proxy.utils import url_based_hash
+from ota_proxy import config as cfg
+from ota_proxy.orm import NULL_TYPE
+from ota_proxy.ota_cache import CacheMeta, OTACacheDB
+from ota_proxy.utils import url_based_hash
 
 logger = logging.getLogger(__name__)
 
 
 class TestORM:
     @pytest.fixture(autouse=True)
     def create_table_defs(self):
-        from otaclient.ota_proxy.orm import NULL_TYPE, ColumnDescriptor, ORMBase
+        from ota_proxy.orm import NULL_TYPE, ColumnDescriptor, ORMBase
 
         @dataclass
         class TableCls(ORMBase):

tests/test_ota_proxy/test_ota_cache.py

@@ -21,10 +21,10 @@
 
 import pytest
 
-from otaclient.ota_proxy import config as cfg
-from otaclient.ota_proxy.db import CacheMeta, OTACacheDB
-from otaclient.ota_proxy.ota_cache import CachingRegister, LRUCacheHelper
-from otaclient.ota_proxy.utils import url_based_hash
+from ota_proxy import config as cfg
+from ota_proxy.db import CacheMeta, OTACacheDB
+from ota_proxy.ota_cache import CachingRegister, LRUCacheHelper
+from ota_proxy.utils import url_based_hash
 
 logger = logging.getLogger(__name__)