From e111be5b8cd7e716693e20806225a2f20ad7dc16 Mon Sep 17 00:00:00 2001 From: kshitijrajsharma Date: Sat, 9 Mar 2024 09:10:41 +0545 Subject: [PATCH] fix(dockerfile): Avoids copying tippicone for source code level changes --- Dockerfile | 71 ++++++++++++++++++--------------- docs/src/installation/docker.md | 48 ++++++++++++++++------ 2 files changed, 74 insertions(+), 45 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca79a03b..eaef19d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,31 @@ FROM docker.io/python:${PYTHON_VERSION}-slim-bookworm as base ARG MAINTAINER=sysadmin@hotosm.org ENV DEBIAN_FRONTEND=noninteractive -FROM base as builder +FROM base as runner + +WORKDIR /home/appuser +ENV PIP_NO_CACHE_DIR=1 +ENV PYTHONUNBUFFERED=1 +ENV PATH="/home/appuser/.local/bin:$PATH" +ENV PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_VERSION/site-packages" + +# Install runtime dependencies +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get --no-install-recommends -y install libpq5 gdal-bin \ + && apt-get -y autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +FROM ghcr.io/hotosm/tippecanoe:main as tippecanoe-builder + +FROM runner as with-tippecanoe +COPY --from=tippecanoe-builder /usr/local/bin/tippecanoe* /usr/local/bin/ +COPY --from=tippecanoe-builder /usr/local/bin/tile-join /usr/local/bin/ + +# Builder stage , python dependencies and project setup +FROM base as python-builder ENV PIP_NO_CACHE_DIR=1 ENV PYTHONUNBUFFERED=1 @@ -23,49 +47,30 @@ COPY pyproject.toml . COPY requirements.txt . COPY README.md . COPY LICENSE . + RUN pip install --user --no-cache-dir --upgrade pip setuptools wheel\ && pip install --user --no-cache-dir GDAL=="$(gdal-config --version)" \ && pip install --user --no-cache-dir -r requirements.txt -RUN pip install --user -e . +RUN python setup.py install --user -FROM base as runner -WORKDIR /home/appuser -ENV PIP_NO_CACHE_DIR=1 -ENV PYTHONUNBUFFERED=1 -ENV PATH="/home/appuser/.local/bin:$PATH" -ENV PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_VERSION/site-packages" -RUN apt-get update \ - && apt-get -y upgrade \ - && apt-get --no-install-recommends -y install libpq5 gdal-bin \ - && apt-get -y autoremove \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +FROM with-tippecanoe as prod +COPY --from=python-builder /root/.local /home/appuser/.local -COPY --from=builder /root/.local /home/appuser/.local -COPY README.md . +RUN useradd --system --uid 900 --home-dir /home/appuser --shell /bin/false appuser \ + && chown -R appuser:appuser /home/appuser -# Enable this if you are using config.txt -# COPY config.txt ./config.txt +USER appuser +# API and source code, changes here don't invalidate previous layers , You can overwrite this block with -v + +# Copy config.txt if you have your configuration setup in config +# COPY config.txt . +COPY README.md . COPY setup.py . COPY pyproject.toml . COPY API/ ./API/ COPY src/ ./src/ -# Use a separate stage to pull the tippecanoe image -FROM ghcr.io/hotosm/tippecanoe:main as tippecanoe-builder - -FROM runner as prod - -# Copy tippecanoe binaries from the tippecanoe stage -COPY --from=tippecanoe-builder /usr/local/bin/tippecanoe* /usr/local/bin/ -COPY --from=tippecanoe-builder /usr/local/bin/tile-join /usr/local/bin/ - -RUN useradd --system --uid 900 --home-dir /home/appuser --shell /bin/false appuser \ - && chown -R appuser:appuser /home/appuser - -USER appuser -# CMD ["/bin/bash"] -CMD ["uvicorn", "API.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000", "--no-use-colors", "--proxy-headers"] \ No newline at end of file +CMD ["uvicorn", "API.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000", "--no-use-colors", "--proxy-headers"] diff --git a/docs/src/installation/docker.md b/docs/src/installation/docker.md index 08968496..cffa1d49 100644 --- a/docs/src/installation/docker.md +++ b/docs/src/installation/docker.md @@ -64,7 +64,7 @@ docker-compose up -d --build OR -### Run Docker without docker compose +### Run Docker without docker compose for development - Build your image ``` @@ -72,23 +72,47 @@ docker build -t rawdataapi:latest . ``` - Run API ``` -docker run -d -p 8000:8000 --name rawdatapi rawdataapi:latest +docker run --name rawdataapi -p 8000:8000 \ + -v "$(pwd)/API:/home/appuser/API" \ + -v "$(pwd)/src:/home/appuser/src" \ + -v "$(pwd)/config.txt:/home/appuser/config.txt" \ + rawdataapi:latest \ + uvicorn API.main:app --reload --host "0.0.0.0" --port "8000" --no-use-colors --proxy-headers ``` -Run container with `.env` file +- Run Workers ``` -docker run --env-file ./.env -d -p 8000:8000 --name rawdatapi rawdataapi:latest +docker run --name rawdata-worker \ + -v "$(pwd)/API:/home/appuser/API" \ + -v "$(pwd)/src:/home/appuser/src" \ + -v "$(pwd)/config.txt:/home/appuser/config.txt" \ + rawdataapi:latest \ + celery --app API.api_worker worker --loglevel=INFO --queues="raw_ondemand" -n 'default_worker' ``` - -- Run Workers +- Run Flower on port 5555 ``` -docker run -it rawdataapi:latest celery --app API.api_worker worker --loglevel=INFO --queues="raw_ondemand" -n 'default_worker' +docker run --name rawdata-flower -p 5555:5555 \ + -v "$(pwd)/API:/home/appuser/API" \ + -v "$(pwd)/src:/home/appuser/src" \ + -v "$(pwd)/config.txt:/home/appuser/config.txt" \ + rawdataapi:latest \ + celery --broker=redis://redis:6379// --app API.api_worker flower --port=5555 --queues="raw_daemon,raw_ondemand" ``` -Followi similar for flower + +**Development instruction:** +If you are running Dockerfile only for the API , Have postgresql redis installed on your machine directly then you should change following : + +- Change --broker Host address in flower command (You can use redis if it is docker compsoe container or use `redis://host.docker.internal:6379/0` if you want API container to connect to your localhsot , Follow #troubleshoot section for more) +- Change DB Host & Celery broker url accordingly with the same logic + + +**Note:** + +In above exampel we have attached our working dir to containers along with config.txt for efficiency in development environment only . It is recommended to use proper docker copy as stated in dockerfile and system environement variables instead of config.txt in Production ## Check the servers -By default, Uvicorn would be running on port [8000](http://127.0.0.1:8000/v1/docs), Redis on default port(i.e 6379), Celery with a worker and Flower on port [5000](http://127.0.0.1:5000/). +By default, Uvicorn would be running on port [8000](http://127.0.0.1:8000/v1/docs), Redis on default port(i.e 6379), Celery with a worker and Flower on port [5555](http://127.0.0.1:5555/). - API Documentation @@ -107,10 +131,10 @@ API docs will be displayed like this upon successfull server startup Vist the route below to access the Flower dashboard ``` -http://127.0.0.1:5000/ +http://127.0.0.1:5555/ ``` -Flower [dashboard](http://127.0.0.1:5000/) will look like this on successfull installation with a worker online. +Flower [dashboard](http://127.0.0.1:5555/) will look like this on successfull installation with a worker online. (Change the port accordingly to your command) ![image](https://user-images.githubusercontent.com/36752999/191813613-3859522b-ea68-4370-87b2-ebd1d8880d80.png) @@ -120,7 +144,7 @@ Flower [dashboard](http://127.0.0.1:5000/) will look like this on successfull in **NOTE:** If any of the solutions provided below does not work for you, please don't hesistate to open an issue. -- If you can't connect to your local postgres database from `export-tool-api`. +- If you can't connect to your local postgres database from `raw-data-api`. Since _export-tool-api_ is running in docker containers, it means if you setup your `config.txt` file to use a local postgres database on your machine, the database port may not be accessible as `localhost` from the docker containers. To solve this, docker needs to connect to your local network. Try any of these hacks to troubleshoot the issue: