diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a0c4533f5..5cfed8963 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,12 +20,13 @@ jobs: - name: install build deps run: | - python -m pip install -U pip build + python -m pip install -U pip + python -m pip install -U hatch python -m pip install -r proto/requirements.txt - name: build otaclient package run: | - python -m build --wheel --outdir dist/ . + hatch build -t wheel - name: build a proto binary run: | diff --git a/docker/docker-compose_tests.yml b/docker/docker-compose_tests.yml index 8a8a26257..ad574d851 100644 --- a/docker/docker-compose_tests.yml +++ b/docker/docker-compose_tests.yml @@ -8,11 +8,7 @@ services: container_name: ota-test environment: OUTPUT_DIR: /test_result - OTACLIENT_DIR: /ota-client CERTS_DIR: /certs volumes: - - ../pyproject.toml:/ota-client/pyproject.toml:ro - - ../.flake8:/ota-client/.flake8:ro - - ../otaclient:/ota-client/otaclient:ro - - ../tests:/ota-client/tests:ro + - ..:/otaclient_src:ro - ../test_result:/test_result:rw diff --git a/docker/test_base/Dockerfile b/docker/test_base/Dockerfile index 0a13c9915..da57daa1c 100644 --- a/docker/test_base/Dockerfile +++ b/docker/test_base/Dockerfile @@ -4,11 +4,9 @@ ENV DEBIAN_FRONTEND=noninteractive ARG KERNEL_VERSION="5.8.0-53-generic" ARG BASE_IMG_URL="http://cdimage.ubuntu.com/ubuntu-base/releases/20.04/release/ubuntu-base-20.04.1-base-amd64.tar.gz" ARG OTA_METADATA_REPO="https://github.com/tier4/ota-metadata" -ENV OTA_CLIENT_DIR="/ota-client" ENV OTA_IMAGE_SERVER_ROOT="/ota-image" ENV OTA_IMAGE_DIR="${OTA_IMAGE_SERVER_ROOT}/data" ENV CERTS_DIR="/certs" -ENV VENV="${OTA_CLIENT_DIR}/.venv" ENV SPECIAL_FILE="path;adf.ae?qu.er\y=str#fragファイルement" @@ -43,17 +41,10 @@ RUN set -eu; \ # NOTE: include special identifiers #?; into the pathname RUN echo -n "${SPECIAL_FILE}" > "${OTA_IMAGE_DIR}/${SPECIAL_FILE}" -# install otaclient dependencies -COPY ./otaclient/requirements.txt /tmp/requirements.txt -COPY ./tests/requirements.txt /tmp/requirements-dev.txt +# install hatch RUN set -eu; \ - python3 -m venv ${VENV}; \ - source ${VENV}/bin/activate; \ python3 -m pip install --no-cache-dir -q -U pip; \ - python3 -m pip install \ - --no-cache-dir -q \ - -r /tmp/requirements.txt \ - -r /tmp/requirements-dev.txt + python3 -m pip install --no-cache-dir -U hatch # generate test certs and sign key COPY --chmod=755 ./tests/keys/gen_certs.sh /tmp/certs/ @@ -98,5 +89,4 @@ RUN set -eu; \ COPY ./docker/test_base/entry_point.sh /entry_point.sh RUN chmod +x /entry_point.sh -WORKDIR ${OTA_CLIENT_DIR} ENTRYPOINT [ "/entry_point.sh" ] diff --git a/docker/test_base/entry_point.sh b/docker/test_base/entry_point.sh index c29a2d4fa..014336a64 100644 --- a/docker/test_base/entry_point.sh +++ b/docker/test_base/entry_point.sh @@ -1,32 +1,24 @@ #!/bin/bash set -eu -OTA_CLIENT_DIR="${OTACLIENT_DIR:-/ota-client}" +TEST_ROOT=/test_root +OTACLIENT_SRC=/otaclient_src OUTPUT_DIR="${OUTPUT_DIR:-/test_result}" CERTS_DIR="${CERTS_DIR:-/certs}" -VENV="${OTA_CLIENT_DIR}/.venv" + +# copy the source code as source is read-only +cp -R "${OTACLIENT_SRC}" "${TEST_ROOT}" # copy the certs generated in the docker image to otaclient dir echo "setup certificates for testing..." -cd ${OTA_CLIENT_DIR} +cd ${TEST_ROOT} mkdir -p ./certs cp -av ${CERTS_DIR}/root.pem ./certs/1.root.pem cp -av ${CERTS_DIR}/interm.pem ./certs/1.interm.pem -source ${VENV}/bin/activate - -# setup dependencies unconditionally -# NOTE: if we only do minor modified to the requirements.txt, -# we don't have to rebuild the image, just call pip here -APP_DEPENDENCIES="${OTA_CLIENT_DIR}/otaclient/requirements.txt" -[ -f "$APP_DEPENDENCIES" ] && echo "setup app dependencies..." && \ - python3 -m pip install --no-cache-dir -q -r $APP_DEPENDENCIES -TESTS_DEPENDENCIES="${OTA_CLIENT_DIR}/tests/requirements.txt" -[ -f "$TESTS_DEPENDENCIES" ] && echo "setup tests dependencies..." && \ - python3 -m pip install --no-cache-dir -q -r $TESTS_DEPENDENCIES - # exec the input params echo "execute test with coverage" -cd ${OTA_CLIENT_DIR} -coverage run -m pytest --junit-xml=${OUTPUT_DIR}/pytest.xml ${@:-} -coverage xml -o ${OUTPUT_DIR}/coverage.xml +cd ${TEST_ROOT} +hatch env create dev +hatch run dev:coverage run -m pytest --junit-xml=${OUTPUT_DIR}/pytest.xml ${@:-} +hatch run dev:coverage xml -o ${OUTPUT_DIR}/coverage.xml diff --git a/pyproject.toml b/pyproject.toml index 3569e108d..41e11a020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [build-system] -build-backend = "setuptools.build_meta" +build-backend = "hatchling.build" requires = [ - "setuptools>=61", - "setuptools_scm[toml]>=6.4", + "hatch-vcs", + "hatchling>=1.20", ] [project] @@ -21,27 +21,54 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dynamic = [ - "dependencies", - "optional-dependencies", "version", ] +dependencies = [ + "aiofiles==22.1", + "aiohttp<3.10.0,>=3.9.5", + "cryptography<43.0.0,>=42.0.4", + "grpcio<1.54.0,>=1.53.2", + "protobuf<4.22.0,>=4.21.12", + "pydantic==2.7", + "pydantic-settings==2.2.1", + "pyOpenSSL==24.1", + "PyYAML>=3.12", + "requests<2.32.0,>=2.31", + 'typing_extensions>=4.6.3; python_version < "3.11"', + "urllib3<2.0.0,>=1.26.8", + "uvicorn[standard]==0.20", + "zstandard==0.18", +] +[project.optional-dependencies] +dev = [ + "black", + "coverage", + "flake8", + "isort", + "pytest==7.1.2", + "pytest-asyncio==0.21", + "pytest-mock==3.8.2", + "requests-mock", +] [project.urls] Source = "https://github.com/tier4/ota-client" -[tool.setuptools.dynamic] -dependencies = { file = ["otaclient/requirements.txt"] } +[tool.hatch.version] +source = "vcs" + +[tool.hatch.build.hooks.vcs] +version-file = "src/_otaclient_version.py" -[tool.setuptools.dynamic.optional-dependencies] -test = { file = ["tests/requirements.txt"] } +[tool.hatch.build.targets.sdist] +exclude = ["/tools"] -[tool.setuptools.packages.find] -include = ["otaclient*"] -namespaces = false +[tool.hatch.build.targets.wheel] +only-include = ["src"] +sources = ["src"] -[tool.setuptools_scm] -write_to = "otaclient/_version.py" -version_scheme = "post-release" -local_scheme = "no-local-version" +[tool.hatch.envs.dev] +type = "virtual" +features = ["dev"] [tool.black] line-length = 88 diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 000000000..4fc778335 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,2 @@ +# generated version file by build +_otaclient_version.py diff --git a/src/otaclient/__init__.py b/src/otaclient/__init__.py index aa32a12b6..c335daf64 100644 --- a/src/otaclient/__init__.py +++ b/src/otaclient/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. try: - from otaclient._version import __version__, version + from _otaclient_version import __version__, version except ImportError: # unknown version version = __version__ = "0.0.0" diff --git a/src/otaclient/_utils/typing.py b/src/otaclient/_utils/typing.py index 451ce4f19..941935f93 100644 --- a/src/otaclient/_utils/typing.py +++ b/src/otaclient/_utils/typing.py @@ -15,6 +15,7 @@ from __future__ import annotations +from enum import Enum from pathlib import Path from typing import Any, Callable, TypeVar, Union @@ -22,7 +23,7 @@ from typing_extensions import Annotated, ParamSpec P = ParamSpec("P") -T = TypeVar("T") +T = TypeVar("T", bound=Enum) RT = TypeVar("RT") StrOrPath = Union[str, Path] diff --git a/src/otaclient/app/downloader.py b/src/otaclient/app/downloader.py index 34ef9190e..173d9b1a3 100644 --- a/src/otaclient/app/downloader.py +++ b/src/otaclient/app/downloader.py @@ -48,8 +48,8 @@ from urllib3.response import HTTPResponse from urllib3.util.retry import Retry +from ota_proxy import OTAFileCacheControl from otaclient._utils import copy_callable_typehint -from otaclient.ota_proxy import OTAFileCacheControl from .common import wait_with_backoff from .configs import config as cfg diff --git a/src/otaclient/app/log_setting.py b/src/otaclient/app/log_setting.py index a60629685..44948b623 100644 --- a/src/otaclient/app/log_setting.py +++ b/src/otaclient/app/log_setting.py @@ -100,5 +100,5 @@ def configure_logging(): # NOTE: "otaclient" logger will be the root logger for all loggers name # starts with "otaclient.", and the settings will affect its child loggers. # For example, settings for "otaclient" logger will also be effective to - # "otaclient.app.*" logger and "otaclient.ota_proxy.*" logger. + # "otaclient.app.*" logger and "ota_proxy.*" logger. _otaclient_logger.addHandler(ch) diff --git a/src/otaclient/app/ota_client_stub.py b/src/otaclient/app/ota_client_stub.py index ff932c0cd..e2cf6c3ef 100644 --- a/src/otaclient/app/ota_client_stub.py +++ b/src/otaclient/app/ota_client_stub.py @@ -28,10 +28,10 @@ from typing_extensions import Self +from ota_proxy import OTAProxyContextProto +from ota_proxy import config as local_otaproxy_cfg +from ota_proxy import subprocess_otaproxy_launcher from otaclient.configs.ecu_info import ECUContact -from otaclient.ota_proxy import OTAProxyContextProto -from otaclient.ota_proxy import config as local_otaproxy_cfg -from otaclient.ota_proxy import subprocess_otaproxy_launcher from . import log_setting from .boot_control._common import CMDHelperFuncs @@ -65,7 +65,7 @@ def __init__( self._external_cache_dev_mp = external_cache_dev_mp self._external_cache_data_dir = external_cache_path - self.logger = logging.getLogger("otaclient.ota_proxy") + self.logger = logging.getLogger("ota_proxy") @property def extra_kwargs(self) -> Dict[str, Any]: @@ -85,9 +85,9 @@ def _subprocess_init(self): # configure logging for otaproxy subprocess # NOTE: on otaproxy subprocess, we first set log level of the root logger # to CRITICAL to filter out third_party libs' logging(requests, urllib3, etc.), - # and then set the otaclient.ota_proxy logger to DEFAULT_LOG_LEVEL + # and then set the ota_proxy logger to DEFAULT_LOG_LEVEL log_setting.configure_logging() - otaproxy_logger = logging.getLogger("otaclient.ota_proxy") + otaproxy_logger = logging.getLogger("ota_proxy") otaproxy_logger.setLevel(cfg.DEFAULT_LOG_LEVEL) self.logger = otaproxy_logger diff --git a/src/otaclient/app/ota_metadata.py b/src/otaclient/app/ota_metadata.py index d0c2651cc..b69a99241 100644 --- a/src/otaclient/app/ota_metadata.py +++ b/src/otaclient/app/ota_metadata.py @@ -71,7 +71,7 @@ from OpenSSL import crypto from typing_extensions import Self -from otaclient.ota_proxy import OTAFileCacheControl +from ota_proxy import OTAFileCacheControl from .common import RetryTaskMap, get_backoff, urljoin_ensure_base from .configs import config as cfg diff --git a/src/otaclient/requirements.txt b/src/otaclient/requirements.txt deleted file mode 100644 index a8f593b6c..000000000 --- a/src/otaclient/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -pyOpenSSL==24.1.0 -cryptography>=42.0.4, <43.0.0 -grpcio>=1.53.2, <1.54.0 -protobuf==4.21.12 -PyYAML>=3.12 -requests==2.31.0 -urllib3>=1.26.8, <2.0.0 -uvicorn[standard]==0.20.0 -aiohttp>=3.9.5, <3.10.0 -aiofiles==22.1.0 -zstandard==0.18.0 -typing_extensions==4.6.3 -pydantic==2.7.0 -pydantic-settings==2.2.1 diff --git a/tests/conftest.py b/tests/conftest.py index b856edd93..c923caf1a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,7 +42,7 @@ class TestConfiguration: OTACLIENT_STUB_MODULE_PATH = "otaclient.app.ota_client_stub" OTACLIENT_SERVICE_MODULE_PATH = "otaclient.app.ota_client_service" OTAMETA_MODULE_PATH = "otaclient.app.ota_metadata" - OTAPROXY_MODULE_PATH = "otaclient.ota_proxy" + OTAPROXY_MODULE_PATH = "ota_proxy" CREATE_STANDBY_MODULE_PATH = "otaclient.app.create_standby" MAIN_MODULE_PATH = "otaclient.app.main" diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index ab1a57093..000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -pytest==7.1.2 -pytest-asyncio==0.21.0 -pytest-mock==3.8.2 -pytest-cov==3.0.0 -black==22.3.0 -flake8==4.0.1 -requests-mock==1.10.0 diff --git a/tests/test_ota_client_stub.py b/tests/test_ota_client_stub.py index e16293d1a..77734df46 100644 --- a/tests/test_ota_client_stub.py +++ b/tests/test_ota_client_stub.py @@ -24,6 +24,8 @@ import pytest from pytest_mock import MockerFixture +from ota_proxy import OTAProxyContextProto +from ota_proxy.config import Config as otaproxyConfig from otaclient.app.ota_client import OTAServicer from otaclient.app.ota_client_call import OtaClientCall from otaclient.app.ota_client_stub import ( @@ -34,8 +36,6 @@ from otaclient.app.proto import wrapper from otaclient.configs.ecu_info import ECUInfo, parse_ecu_info from otaclient.configs.proxy_info import ProxyInfo, parse_proxy_info -from otaclient.ota_proxy import OTAProxyContextProto -from otaclient.ota_proxy.config import Config as otaproxyConfig from tests.conftest import cfg from tests.utils import compare_message diff --git a/tests/test_ota_proxy/test_cache_control_header.py b/tests/test_ota_proxy/test_cache_control_header.py index 85242d162..30adbd990 100644 --- a/tests/test_ota_proxy/test_cache_control_header.py +++ b/tests/test_ota_proxy/test_cache_control_header.py @@ -17,7 +17,7 @@ import pytest -from otaclient.ota_proxy import OTAFileCacheControl +from ota_proxy import OTAFileCacheControl @pytest.mark.parametrize( diff --git a/tests/test_ota_proxy/test_cachedb.py b/tests/test_ota_proxy/test_cachedb.py index 7c3d3c3fd..4f5992ee4 100644 --- a/tests/test_ota_proxy/test_cachedb.py +++ b/tests/test_ota_proxy/test_cachedb.py @@ -22,10 +22,10 @@ import pytest -from otaclient.ota_proxy import config as cfg -from otaclient.ota_proxy.orm import NULL_TYPE -from otaclient.ota_proxy.ota_cache import CacheMeta, OTACacheDB -from otaclient.ota_proxy.utils import url_based_hash +from ota_proxy import config as cfg +from ota_proxy.orm import NULL_TYPE +from ota_proxy.ota_cache import CacheMeta, OTACacheDB +from ota_proxy.utils import url_based_hash logger = logging.getLogger(__name__) @@ -33,7 +33,7 @@ class TestORM: @pytest.fixture(autouse=True) def create_table_defs(self): - from otaclient.ota_proxy.orm import NULL_TYPE, ColumnDescriptor, ORMBase + from ota_proxy.orm import NULL_TYPE, ColumnDescriptor, ORMBase @dataclass class TableCls(ORMBase): diff --git a/tests/test_ota_proxy/test_ota_cache.py b/tests/test_ota_proxy/test_ota_cache.py index 783826bae..26dec048c 100644 --- a/tests/test_ota_proxy/test_ota_cache.py +++ b/tests/test_ota_proxy/test_ota_cache.py @@ -21,10 +21,10 @@ import pytest -from otaclient.ota_proxy import config as cfg -from otaclient.ota_proxy.db import CacheMeta, OTACacheDB -from otaclient.ota_proxy.ota_cache import CachingRegister, LRUCacheHelper -from otaclient.ota_proxy.utils import url_based_hash +from ota_proxy import config as cfg +from ota_proxy.db import CacheMeta, OTACacheDB +from ota_proxy.ota_cache import CachingRegister, LRUCacheHelper +from ota_proxy.utils import url_based_hash logger = logging.getLogger(__name__) diff --git a/tests/test_ota_proxy/test_ota_proxy_server.py b/tests/test_ota_proxy/test_ota_proxy_server.py index 7aa07eb8c..c38015cda 100644 --- a/tests/test_ota_proxy/test_ota_proxy_server.py +++ b/tests/test_ota_proxy/test_ota_proxy_server.py @@ -27,9 +27,9 @@ import pytest import uvicorn +from ota_proxy.utils import url_based_hash from otaclient.app.ota_metadata import parse_regulars_from_txt from otaclient.app.proto.wrapper import RegularInf -from otaclient.ota_proxy.utils import url_based_hash from tests.conftest import ThreadpoolExecutorFixtureMixin, cfg logger = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class TestOTAProxyServer(ThreadpoolExecutorFixtureMixin): async def setup_ota_proxy_server(self, tmp_path: Path, request): import uvicorn - from otaclient.ota_proxy import App, OTACache + from ota_proxy import App, OTACache ota_cache_dir = tmp_path / "ota-cache" ota_cache_dir.mkdir(parents=True, exist_ok=True) @@ -173,7 +173,7 @@ async def test_download_file_with_special_fname(self): """ import sqlite3 - from otaclient.ota_proxy import config as cfg + from ota_proxy import config as cfg # ------ get the special file via otaproxy from the ota image server ------ # # --- execution --- # @@ -295,7 +295,7 @@ class TestOTAProxyServerWithoutCache(ThreadpoolExecutorFixtureMixin): async def setup_ota_proxy_server(self, tmp_path: Path): import uvicorn - from otaclient.ota_proxy import App, OTACache + from ota_proxy import App, OTACache ota_cache_dir = tmp_path / "ota-cache" ota_cache_dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_ota_proxy/test_subprocess_launch_otaproxy.py b/tests/test_ota_proxy/test_subprocess_launch_otaproxy.py index 512a2bed3..552f2ad44 100644 --- a/tests/test_ota_proxy/test_subprocess_launch_otaproxy.py +++ b/tests/test_ota_proxy/test_subprocess_launch_otaproxy.py @@ -14,11 +14,10 @@ import time -from functools import partial from pathlib import Path from typing import Any, Dict -from otaclient.ota_proxy import OTAProxyContextProto, subprocess_otaproxy_launcher +from ota_proxy import OTAProxyContextProto, subprocess_otaproxy_launcher class _DummyOTAProxyContext(OTAProxyContextProto):