From f666876b29d9b570d546722ae35cbc5d61980478 Mon Sep 17 00:00:00 2001 From: febrezo Date: Sat, 31 Dec 2022 00:42:08 +0100 Subject: [PATCH] Update cron approach and docs --- .env => .env.template | 1 + .gitignore | 10 +- Dockerfile | 13 +- README.md | 154 +++++++++--------- app/run.py | 4 +- .../{cron.d/service-crontab => crontabs/root} | 0 config/dev/service_variables.env | 4 - config/dev/variables.env | 2 + config/pre/cron.d/service-crontab | 25 --- config/pre/cron.d/service-crontab.template | 25 --- config/pre/service_variables.env | 4 - config/pre/service_variables.env.template | 4 - config/pre/variables.env.template | 2 + config/pro/cron.d/service-crontab.template | 25 --- config/pro/service_variables.env.templatae | 4 - config/pro/variables.env.template | 2 + docker-compose.yml | 20 +-- docs/ENVIRONMENT_VARIABLES.md | 30 ++-- docs/LESSONS_LEARNED.md | 153 +++++++++++++++++ docs/PRODUCTION_RECOMMENDATIONS.md | 84 +++++++++- docs/README.md | 1 + docs/VOLUME_MANAGEMENT.md | 16 +- 22 files changed, 356 insertions(+), 227 deletions(-) rename .env => .env.template (96%) rename config/dev/{cron.d/service-crontab => crontabs/root} (100%) delete mode 100644 config/dev/service_variables.env create mode 100644 config/dev/variables.env delete mode 100644 config/pre/cron.d/service-crontab delete mode 100644 config/pre/cron.d/service-crontab.template delete mode 100644 config/pre/service_variables.env delete mode 100644 config/pre/service_variables.env.template create mode 100644 config/pre/variables.env.template delete mode 100644 config/pro/cron.d/service-crontab.template delete mode 100644 config/pro/service_variables.env.templatae create mode 100644 config/pro/variables.env.template create mode 100644 docs/LESSONS_LEARNED.md diff --git a/.env b/.env.template similarity index 96% rename from .env rename to .env.template index 8af53b0..74717ab 100644 --- a/.env +++ b/.env.template @@ -7,6 +7,7 @@ # Check ./docs/ENVIRONMENT_VARIABLES.md and ./docs/VOLUMES.md to understand # the usage of these strings. +APP_NAME=tempered-iron # Environment to use ENVIRONMENT=dev # ============================================================================== diff --git a/.gitignore b/.gitignore index 44587bf..4761d82 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ -pre/*.env -pro/*.env -pre/*-crontab -pro/*-crontab \ No newline at end of file +config/pre/*.env +config/pro/*.env +config/pre/crontabs/* +config/pro/crontabs/* +.env +volumes diff --git a/Dockerfile b/Dockerfile index e0fef97..7b30ead 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,9 @@ FROM python:3.10-alpine -RUN apk update - -# Setting a default crontab for the service -COPY config/dev/cron.d/service-crontab /etc/cron.d/service-crontab - -RUN chmod 0644 /etc/cron.d/service-crontab &&\ - crontab /etc/cron.d/service-crontab +# Installing a blank crontab for the user 'root' in the service +RUN touch /etc/crontabs/root +# Setting permissions to make the file writable only by the owner +RUN chmod 0644 /etc/crontabs/root # Fixing Python deps ADD requirements.txt /tmp/requirements.txt @@ -16,4 +13,4 @@ RUN pip install -r /tmp/requirements.txt RUN mkdir /data ADD app /app WORKDIR /app -ENTRYPOINT ["sh", "-c" , "crond -f"] \ No newline at end of file +ENTRYPOINT ["sh", "-c" , "crond -f"] diff --git a/README.md b/README.md index 191c849..c715814 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,25 @@ This software is licensed using GPLv3. Check the [`COPYING`](COPYING) file for f ## Dependencies -Check that you have Docker and Docker Compose installed in your computer. -Specfic instructions to do so in your OS can be found in the official docs for Docker ([here](https://docs.docker.com/get-docker/)) and Docker Compose ([here](https://docs.docker.com/compose/install/)). +Check that you have Docker installed in your computer. +Specfic instructions to do so in your OS can be found in the official docs for Docker ([here](https://docs.docker.com/get-docker/)). +Note that old versions of Docker will not install Docker Compose as part of Docker. +This means that any reference to `docker compose` (being `compose` a subcommand of `docker`) will have to be substituted by `docker-compose`. +The ouptut will typically look like this: -Once installed, you can check the versions using the following commands: +``` +$ docker compose up +docker: 'compose' is not a docker command. +See 'docker --help' +``` + +In those cases, install Docker Compose as shown [here](https://docs.docker.com/compose/install/) and check the installation like this: ``` $ docker --version Docker version 18.09.0, build 4d60db4 -$ docker-compose --version -docker-compose version 1.23.1, build b02f1306 +$ docker compose --version +docker compose version 1.23.1, build b02f1306 ``` For convenience, the `Dockerfile` will deal with the Python dependencies at building time using `pip`, but any other additional dependencies and images can be used here. @@ -37,55 +46,50 @@ Note that some files in the config folder are deliberately not copied in the Git To start everything, use: ``` -$ docker-compose up +$ docker compose up ``` Note that if you make changes and you want to see them you will need the `--build` option: ``` -$ docker-compose up --build +$ docker compose up --build +$ docker compose up --build +WARNING: The APP_NAME variable is not set. Defaulting to a blank string. +Creating network "tempered-iron_default" with the default driver +Creating volume "dev_data_vol" with local driver +Creating volume "dev_log_vol" with local driver Building main_service -Step 1/10 : FROM python:3.10-alpine - ---> 482b8fa3563e -Step 2/10 : RUN apk update +Sending build context to Docker daemon 208.4kB +Step 1/9 : FROM python:3.10-alpine + ---> 2527f31628e7 +Step 2/9 : RUN touch /etc/crontabs/root ---> Using cache - ---> 39bccfa17f0d -Step 3/10 : COPY config/dev/cron.d/service-crontab /etc/cron.d/service-crontab + ---> 929555f6812e +Step 3/9 : RUN chmod 0644 /etc/crontabs/root ---> Using cache - ---> 11b24ce1300a -Step 4/10 : RUN chmod 0644 /etc/cron.d/service-crontab && crontab /etc/cron.d/service-crontab + ---> b1fe1619fc32 +Step 4/9 : ADD requirements.txt /tmp/requirements.txt ---> Using cache - ---> 5d5a0e2a504c -Step 5/10 : ADD requirements.txt /tmp/requirements.txt + ---> f13593d70c5d +Step 5/9 : RUN pip install -r /tmp/requirements.txt ---> Using cache - ---> d8e86d8bba81 -Step 6/10 : RUN pip install -r /tmp/requirements.txt + ---> 2a63ab14abb4 +Step 6/9 : RUN mkdir /data ---> Using cache - ---> 2e4742159266 -Step 7/10 : RUN mkdir /data + ---> d2331cd7c967 +Step 7/9 : ADD app /app ---> Using cache - ---> 40b54851e289 -Step 8/10 : ADD app /app - ---> 93e29a3a9aa6 -Step 9/10 : WORKDIR /app - ---> Running in 08a379190249 -Removing intermediate container 08a379190249 - ---> 2c703d06f589 -Step 10/10 : ENTRYPOINT ["sh", "-c" , "crond -f"] - ---> Running in ad51f1e83a7d -Removing intermediate container ad51f1e83a7d - ---> 425bae76ea02 - -Successfully built 425bae76ea02 -Successfully tagged temperediron_main_service:latest -Recreating temperediron_main_service_1 ... -Recreating temperediron_main_service_1 ... done -Attaching to temperediron_main_service_1 -main_service_1 | [INFO] tempered_iron @ 2022-04-05 15:45:00,779 | Checking the public IP… -main_service_1 | [DEBUG] tempered_iron @ 2022-04-05 15:45:00,781 | Starting new HTTPS connection (1): ifconfig.me:443 -main_service_1 | [DEBUG] tempered_iron @ 2022-04-05 15:45:01,040 | https://ifconfig.me:443 "GET / HTTP/1.1" 200 14 -main_service_1 | [INFO] tempered_iron @ 2022-04-05 15:45:01,041 | My public IP is: XX.XX.XX.XX -main_service_1 | [INFO] tempered_iron @ 2022-04-05 15:45:01,041 | Closing the application… + ---> 394f5317655b +Step 8/9 : WORKDIR /app + ---> Using cache + ---> 68e4724b652f +Step 9/9 : ENTRYPOINT ["sh", "-c" , "crond -f"] + ---> Using cache + ---> 677d6091020d +Successfully built 677d6091020d +Successfully tagged tempered-iron_main_service:latest +Creating tempered-iron_main_service_1 ... done +Attaching to tempered-iron_main_service_1 ``` Once started, the deployed container will start a cron service to run schedule tasks. @@ -93,19 +97,19 @@ In this context, the scheduled task is as simple as grabbing the public IP addre ## Additional Docker and Docker Compose Tips -To stop the containers, you can always use `docker-compose down` from the project home. -At the same time, to reraise without building, use `docker-compose up`. +To stop the containers, you can always use `docker compose down` from the project home. +At the same time, to reraise without building, use `docker compose up`. Note that this approach will show the output in the terminal. However, running this in detached mode can be useful. ``` -$ docker-compose up -d +$ docker compose up -d ``` -To grab the logs of a specifc container set up using `docker-compose` you can use a specific command: `docker-compose logs ` as follows: +To grab the logs of a specifc container set up using `docker-compose` you can use a specific command: `docker compose logs ` as follows: ``` -$ docker-compose logs +$ docker compose logs Attaching to temperediron_main_service_1 main_service_1 | [INFO] tempered_iron @ 2022-04-05 15:45:00,779 | Checking the public IP… main_service_1 | [DEBUG] tempered_iron @ 2022-04-05 15:45:00,781 | Starting new HTTPS connection (1): ifconfig.me:443 @@ -118,10 +122,10 @@ main_service_1 | [INFO] tempered_iron @ 2022-04-05 15:52:00,964 | Closing the a ``` With this approach, the application will end a return to the prompt. -However, if you prefer to _follow_ the logs, you can use the `-f` or `--follow` as shown with `docker-compose logs --help` so as to let the application wait for new lines: +However, if you prefer to _follow_ the logs, you can use the `-f` or `--follow` as shown with `docker compose logs --help` so as to let the application wait for new lines: ``` -$ docker-compose logs --help +$ docker compose logs --help View output from containers. Usage: logs [options] [SERVICE...] @@ -132,50 +136,41 @@ Options: -t, --timestamps Show timestamps. --tail="all" Number of lines to show from the end of the logs for each container. -$ docker-compose logs -f +$ docker compose logs -f ``` -Note that for debugging purposes you might find useful to enter the container using a shell. -This can be done using `docker exec -it` command once you know the name of the container. -In the aformentioned case, you can see in that the ID of the container is `425bae76ea02` from the output. +Once started, the deployed container will start a cron service to run scheduled tasks as stated in the `/etc/crontabs/root` file within the container. +In this context, the scheduled task is as simple as grabbing the public IP address and logging it both, in the terminal and in a specific log file under `/log` in the container and in a volume mounted in the host. + +### Basic debugging -If you have already run the container, you can check the running ones using `docker ps` to get the ID of the container you want to enter. +Note that when developing the application things may crash. +This WILL happen. +You can enter the container and run manually the application by listing the containers and opening a shell on it. +To do so: + +1. List the containers being run (YOU SHOULD HAVE NOT STOPPED THE CONTAINER BY YOURSELF!): ``` $ docker ps -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -cf5d42a2da7d temperediron_main_service "sh -c 'crond -f' sh…" 2 minutes ago Up 2 minutes temperediron_main_service_1 +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +29a49899accb temperediron_main_service "sh -c 'crond -f' sh…" 21 seconds ago Up 20 seconds temperediron_main_service_1 ``` -In this case, entering the container is as simple as entering the container with a shell in interactive mode (`-it`). -Note that the default shell of Alpine containers is `sh`, but use `bash` if it is available in the Docker image you used as a base image. +2. Identify the `CONTAINER ID`. In this case, `29a49899accb`. + +3. Logging into the container: ``` -$ docker exec -it cf5d42a2da7d sh +$ docker exec -it 29a49899accb sh +/app # whoami +root /app # ls run.py -/app # python3 run.py -[INFO] tempered_iron @ 2022-04-05 15:49:08,761 | Checking the public IP… -[DEBUG] tempered_iron @ 2022-04-05 15:49:08,763 | Starting new HTTPS connection (1): ifconfig.me:443 -[DEBUG] tempered_iron @ 2022-04-05 15:49:08,928 | https://ifconfig.me:443 "GET / HTTP/1.1" 200 14 -[INFO] tempered_iron @ 2022-04-05 15:49:08,930 | My public IP is: XX.XX.XX.XX -[INFO] tempered_iron @ 2022-04-05 15:49:08,930 | Closing the application… -/app # cd /log/ -/log # ls -tempered_iron.log -/log # tail -n 10 tempered_iron.log -[INFO] tempered_iron @ 2022-04-05 15:49:00,727 | Checking the public IP… -[DEBUG] tempered_iron @ 2022-04-05 15:49:00,729 | Starting new HTTPS connection (1): ifconfig.me:443 -[DEBUG] tempered_iron @ 2022-04-05 15:49:00,940 | https://ifconfig.me:443 "GET / HTTP/1.1" 200 14 -[INFO] tempered_iron @ 2022-04-05 15:49:00,942 | My public IP is: XX.XX.XX.XX -[INFO] tempered_iron @ 2022-04-05 15:49:00,942 | Closing the application… -[INFO] tempered_iron @ 2022-04-05 15:49:08,761 | Checking the public IP… -[DEBUG] tempered_iron @ 2022-04-05 15:49:08,763 | Starting new HTTPS connection (1): ifconfig.me:443 -[DEBUG] tempered_iron @ 2022-04-05 15:49:08,928 | https://ifconfig.me:443 "GET / HTTP/1.1" 200 14 -[INFO] tempered_iron @ 2022-04-05 15:49:08,930 | My public IP is: XX.XX.XX.XX -[INFO] tempered_iron @ 2022-04-05 15:49:08,930 | Closing the application… +/app # ``` +You are now in the container! Have fun! ## Additional Tuning @@ -189,4 +184,3 @@ Follow the instructions on [`./PRODUCTION_RECOMMENDATIONS.md`](./docs/PRODUCTION Note that you can find more information about the project in the [`./ENVIRONMENT_VARIABLES`](./docs/ENVIRONMENT_VARIABLES.md) with more information about how to work with environment variables. At the same time in the [`./VOLUME_MANAGEMENT.md`](./docs/VOLUME_MANAGEMENT.md) you can find more information about how volumes are used. - diff --git a/app/run.py b/app/run.py index 91ba203..c4ba1b5 100644 --- a/app/run.py +++ b/app/run.py @@ -5,7 +5,7 @@ # Grabbing environment variables -LOGGER_NAME = os.environ.get("LOGGER_NAME", "Default App Logger") +LOGGER_NAME = os.environ.get("APP_NAME", "tempered-iron") LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") logging.basicConfig( @@ -30,4 +30,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/config/dev/cron.d/service-crontab b/config/dev/crontabs/root similarity index 100% rename from config/dev/cron.d/service-crontab rename to config/dev/crontabs/root diff --git a/config/dev/service_variables.env b/config/dev/service_variables.env deleted file mode 100644 index d86ef30..0000000 --- a/config/dev/service_variables.env +++ /dev/null @@ -1,4 +0,0 @@ -# The name of the logger to be used. Typically, the name of the app, but not required. -LOGGER_NAME=tempered_iron -# One of: DEBUG, INFO, WARNING, ERROR -LOG_LEVEL=DEBUG \ No newline at end of file diff --git a/config/dev/variables.env b/config/dev/variables.env new file mode 100644 index 0000000..7beceb6 --- /dev/null +++ b/config/dev/variables.env @@ -0,0 +1,2 @@ +# One of: DEBUG, INFO, WARNING, ERROR +LOG_LEVEL=DEBUG diff --git a/config/pre/cron.d/service-crontab b/config/pre/cron.d/service-crontab deleted file mode 100644 index 38aeb85..0000000 --- a/config/pre/cron.d/service-crontab +++ /dev/null @@ -1,25 +0,0 @@ -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -# m h dom mon dow command -0 * * * * python3 /app/run.py - - diff --git a/config/pre/cron.d/service-crontab.template b/config/pre/cron.d/service-crontab.template deleted file mode 100644 index 38aeb85..0000000 --- a/config/pre/cron.d/service-crontab.template +++ /dev/null @@ -1,25 +0,0 @@ -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -# m h dom mon dow command -0 * * * * python3 /app/run.py - - diff --git a/config/pre/service_variables.env b/config/pre/service_variables.env deleted file mode 100644 index e678f6c..0000000 --- a/config/pre/service_variables.env +++ /dev/null @@ -1,4 +0,0 @@ -# The name of the logger to be used. Typically, the name of the app, but not required. -LOGGER_NAME=tempered_iron -# One of: DEBUG, INFO, WARNING, ERROR -LOG_LEVEL=INFO \ No newline at end of file diff --git a/config/pre/service_variables.env.template b/config/pre/service_variables.env.template deleted file mode 100644 index a6dfde7..0000000 --- a/config/pre/service_variables.env.template +++ /dev/null @@ -1,4 +0,0 @@ -# The name of the logger to be used. Typically, the name of the app, but not required. -LOGGER_NAME=tempered_iron -# One of: DEBUG, INFO, WARNING, ERROR -LOG_LEVEL=WARNING \ No newline at end of file diff --git a/config/pre/variables.env.template b/config/pre/variables.env.template new file mode 100644 index 0000000..7cd81fc --- /dev/null +++ b/config/pre/variables.env.template @@ -0,0 +1,2 @@ +# One of: DEBUG, INFO, WARNING, ERROR +LOG_LEVEL=INFO diff --git a/config/pro/cron.d/service-crontab.template b/config/pro/cron.d/service-crontab.template deleted file mode 100644 index 38aeb85..0000000 --- a/config/pro/cron.d/service-crontab.template +++ /dev/null @@ -1,25 +0,0 @@ -# Edit this file to introduce tasks to be run by cron. -# -# Each task to run has to be defined through a single line -# indicating with different fields when the task will be run -# and what command to run for the task -# -# To define the time you can provide concrete values for -# minute (m), hour (h), day of month (dom), month (mon), -# and day of week (dow) or use '*' in these fields (for 'any').# -# Notice that tasks will be started based on the cron's system -# daemon's notion of time and timezones. -# -# Output of the crontab jobs (including errors) is sent through -# email to the user the crontab file belongs to (unless redirected). -# -# For example, you can run a backup of all your user accounts -# at 5 a.m every week with: -# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ -# -# For more information see the manual pages of crontab(5) and cron(8) -# -# m h dom mon dow command -0 * * * * python3 /app/run.py - - diff --git a/config/pro/service_variables.env.templatae b/config/pro/service_variables.env.templatae deleted file mode 100644 index 0901a28..0000000 --- a/config/pro/service_variables.env.templatae +++ /dev/null @@ -1,4 +0,0 @@ -# The name of the logger to be used. Typically, the name of the app, but not required. -LOGGER_NAME=tempered_iron -# One of: DEBUG, INFO, WARNING, ERROR -LOG_LEVEL=ERROR \ No newline at end of file diff --git a/config/pro/variables.env.template b/config/pro/variables.env.template new file mode 100644 index 0000000..28f8660 --- /dev/null +++ b/config/pro/variables.env.template @@ -0,0 +1,2 @@ +# One of: DEBUG, INFO, WARNING, ERROR +LOG_LEVEL=WARNING diff --git a/docker-compose.yml b/docker-compose.yml index 92f69be..9f590a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,18 +4,12 @@ services: main_service: build: . command: sh -c "crond -f" - restart: on-failure:5 + #restart: unless-stopped + environment: + - APP_NAME=${APP_NAME} env_file: - - ./config/${ENVIRONMENT:-dev}/service_variables.env + - ./config/${ENVIRONMENT:-dev}/variables.env volumes: - - data_vol:/data - - log_vol:/log - - ./config/${ENVIRONMENT:-dev}/cron.d:/etc/cron.d - -volumes: - data_vol: - name: ${ENVIRONMENT:-dev}_data_vol - driver: local - log_vol: - name: ${ENVIRONMENT:-dev}_log_vol - driver: local \ No newline at end of file + - ./volumes/data:/data + - ./volumes/log:/log + - ./config/${ENVIRONMENT:-dev}/crontabs:/etc/crontabs diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 3c4dd71..a5bf3cc 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -1,8 +1,7 @@ # Environment Variables Environment variables are a typical way of configuring the different environments. -In this file [we](mailto:felix.brezofernandez@telefonica.com) propose an approach based to set the environment using different environment folders. -These folders in this directory contain the environment variable files for the different services that are being deployed using `docker-compose`. +These folders in this directory contain the environment variable files for the different services that are being deployed using the `docker-compose.yml` file. ## About the Different Environments @@ -12,32 +11,33 @@ By default, two different environment are included: - `dev`. Used for quick and fast deployments. Simply by following the initial instructions, a new local environment can be installed. - `pro`. Used for production. Note that production-ready variables are deliberately ignored in the folder. Thus, `template` files are used in this folder. Note that to avoid the credential leakage, production environment variables SHOULD be avoided in the platform. -Preferably, for bigger projects, additional environments will be created such as `qa`/`testing`, `security` or `pre`. -Note that `pro/service_variables.env` and `pre/service_variables.env` are ignored in the `.gitignore` file by default But any other environment SHOULD be manually added to the `.gitignore` to be sure that no additional file is being added to Docker by mistake. +Preferably, for bigger projects, additional environments may be created such as `qa`, `testing`, `security` or `pre`. +Note that `pro/*.env` and `pre/*.env` files are ignored in the `.gitignore` file by default. +These files SHOULD be added to the `.gitignore` to be sure that no additional file is being added to Docker by mistake. ## The Usage of the `.env` File The `.env` file in the main folder is not a standard environment variable file. -Environment variables to be used within a container SHOULD not be defined here, since this file is used natively by `docker-compose` to replace the values of the string within that file. +Environment variables to be used within a container SHOULD not be defined here, since this file is used natively by Docker Compose to replace the values of the string within that file. -For example, the `.env` file can be used to tell `docker-compose` which environment variables to use in the `docker-compose.yml` deployment without even changing the `docker-compose.yml` file and the name of the volumes and services that will be created. +For example, the `.env` file can be used to tell `docker compose` which environment variables to use in the `docker-compose.yml` deployment without even changing the `docker-compose.yml` file and the name of the volumes and services that will be created. The initial contents of this file make reference to the following strings: -- `ENVIRONMENT`. Used to define the configuration to be loaded in the created containers. +- `APP_NAME`. Sets the name for the application. Typically used in the logging methods. +- `ENVIRONMENT`. Used to define the configuration to be loaded in the created containers. The `docker-compose.yml` file will try to locate environment files under the `./config//` folder. Note that you WILL need to update these files using the template files stored in the folder as a reference. +Alternative varuavkes may include, for example, access tokens to be able to reach private packages in Github. -``` -ENVIRONMENT=dev -``` +- `GIT_ACCESS_TOKEN`. Used when a private package from Github needs to be grabbed. -The `docker-compose.yml` file is already build with this approach in mind as explained in the +The `docker-compose.yml` file is already build with this approach in mind. -## Details +## Environment Variables Usage -This section defines the environment variables that are being used in this full dockerised environment. +This section defines the environment variables that are being used in the service. +Note that you will probably want to set in this file things like API keys, remote hosts and similar things. -- `LOGGER_NAME`. Sets the name for the logger. - `LOG_LEVEL`. Sets the logging level. It SHOULD be one of the following: `DEBUG`, `INFO`, `WARNING` or `ERROR`. In Python scripts, note that the value of an environment variable named `LOGGER_NAME` can be grabbed as follows: @@ -47,4 +47,4 @@ import os VALUE = os.environ.get("LOGGER_NAME", "Logger") ``` -Note that, using `os.environ.get("", ""]` to set default values if they are not present and avoid the fact of having to deal with exceptions. \ No newline at end of file +Note that, using `os.environ.get("", ""]` to set default values if they are not present and avoid the fact of having to deal with exceptions. diff --git a/docs/LESSONS_LEARNED.md b/docs/LESSONS_LEARNED.md new file mode 100644 index 0000000..1012dce --- /dev/null +++ b/docs/LESSONS_LEARNED.md @@ -0,0 +1,153 @@ +# Lessons Learned + +There are a few concepts about Docker containers and volumes, Pip, Python, Cron and basic GNU/Linux tips that you may learn from using this repository. +The following is a non-exhaustive list of concepts that are widely used and that the author thinks that may be useful for those who want to understand the hows and whys about this repository. + +## Python and Pip + +Typically, dependencies in Python are managed using Pip. +That's why many projects will ask you to install a dependency (e. g., the `requests` package) using something like this: + +``` +$ pip3 install requests +``` + +These dependencies are automatically downloaded from Python Package Index (the well-known PyPI) by default. +However, when there are a lot of requirements, a `requirements.txt` file is more appropriate to avoid having to install all dependencies manually. +That's why in many places fixing dependencies will look like this to go through a series of dependencies: + +``` +$ pip3 install -r requirements.txt +``` + +There is a minor trick performed in this project regarding the requirements. +The project eases the installation of a package which is stored in Github as a private repository so the building process will need to have read acess to read it. +That's why there is a strange and long line in this file. + + +Note that this project uses Python 3. +In fact, since it uses f-strings (check [this](https://docs.python.org/3/tutorial/inputoutput.html) to understand their usage), only Python 3.6+ can be used. + +The project also receives configuration variables as OS environment variables. +This is very useful to share configurations between services. +Note that OS environment variables can be easily grabbed in Python using: + +``` +import os +my_vars = os.environ +print(my_vars) +``` + +At the same time, it is recommended to avoid printing logging lines. +The sample project uses a basic logging manager that logs things in a given format and sends the output to a logging file. +Both concepts, environment variables and logging are demo-ed in the `run.py` file under the sample `app` folder. + +``` +import logging +import os + +… + +# Grabbing environment variables +LOGGER_NAME = os.environ.get("APP_NAME", "tempered-iron") +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") + +logging.basicConfig( + level=LOG_LEVEL, + format=f"[%(levelname)s] {LOGGER_NAME} @ %(asctime)s | %(message)s", + handlers=[ + logging.FileHandler(f'/log/{LOGGER_NAME}.log'), + logging.StreamHandler() + ] +) + +… + +def main(): + logging.info("Checking the public IP…") +… +``` + +## Docker + +One of the problems of managing different environments are dependencies and conflicts between them when being installed in different systems. +That's where Docker (and the `Dockerfile` files) appear. +In a `Dockerfile` we are telling Docker which is the base image to use (e. g., `FROM python:3.10-alpine`, `FROM ubuntu:latest`, etc.). +To that image we will be able to copy (both, `COPY` or `ADD` do basically the same, google it for the differences) the files we want to have in the built image from our host system. +Once everything desired is inside, we can run commands at building time (`RUN pip install -r requirements`) to prepare the image for the deployment. + +Building a Docker image is the first step, but after building it we have to start it. +That is where the `docker-compose.yml` file can help a lot. +In a `docker-compose.yml` we can define the services we want to start like our own custom image or one taken from third parties to start a MySQL database, ElasticSearch or Mongo DB amongst many many others using, simply, a YAML syntax. + +By default, when launching a `docker-compose.yml` file, `docker-compose` (or `docker compose` in recent versions) will try to locate a hidden `.env` file in the folder. +If found, it will read the variables on it a replace the values in the file with that value. + +For example, if the `.env` file contains: + +``` +APP_NAME=hacking_app +``` + +And the `docker-compose.yml` contains the following lines: +``` + - APP_NAME=${APP_NAME} +``` + +This line will be interpreted by Docker as: + +``` + - APP_NAME=hacking_app +``` + +Although this is a way of sharing values between different services, for security reasons, it is commonly not a good practice to share amongst all the services all the variables. +For example, a frontend may not need to have the DB credentials if the read/write operations are performed by some kind of middleware. + +The point is that the `docker-compose.yml` file can help us to propagate specific environment variables to the different services using different approaches. +For example: + +- Using the `environment:` keyword we can set an environment variable manually. + +``` + env_file: + - ./config/${ENVIRONMENT:-dev}/variables.env +``` + +- Using the `env_file:` keyword we can read them from a file and propagate them to the services we want. + +``` + env_file: + - ./config/${ENVIRONMENT:-dev}/variables.env +``` + +### Volumes + +Once run, a container builds its OS and lives as a standard GNU/Linux OS with a given particularity: it is ephemeral. +Whenever you stop the container (or use `docker compose down`), things simply get vanished. + +Of course, there is a way of persisting things: the volumes. +With a volume, in the `docker-compose.yml` file we can map a local folder in the host system onto a particular path in the `container`. +Take a look at these lines: + + +Of course, there is a way of persisting things: the volumes. +With a volume, in the `docker-compose.yml` file we can map a local folder in the host system onto a particular path in the `container`. +Take a look at these lines: + +``` + volumes: + - ./volumes/data:/data + - ./volumes/log:/log +``` + +They mean that the `/data` and the `/log` folders INSIDE the container will be mapped onto newly created folders in the host under the relative paths `./volumes/data` and `./volumes/log`. +There are other ways to do this (google Docker volumes in production for more info), but this is basically how you can persist data: by making your app write things under these folders. + + +## Documentation + +Documentation files are written in Markdown and they have a `.md` extension. +The names of the files are typically written in capital letters (i. e., `README.md`, `HACKING.md`, etc.). +Most of the reachable documentation folders contain a `README.md` file in their root as a reference file because Github, Gitlab and others will typically render the contents in these files by default when exploring the folder. + +If you want to improve the way in which you write documentation in Markdown (bold, italics, links, images, tables, etc.), please refer to the [official docs](https://docs.github.com/es/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax) at Github. diff --git a/docs/PRODUCTION_RECOMMENDATIONS.md b/docs/PRODUCTION_RECOMMENDATIONS.md index 17831e6..527e70d 100644 --- a/docs/PRODUCTION_RECOMMENDATIONS.md +++ b/docs/PRODUCTION_RECOMMENDATIONS.md @@ -7,12 +7,17 @@ So as to move to a production-ready environment, some recommendations are define This will bring a default installation WITHOUT production variables. 1. Set the `ENVIRONMENT` variable in the `.env` to `pre`, `pro` or whatever other environment you are using. This will create the volumens for the environment automatically when launching `docker-compose`. -2. Create or update the `config//service_variables.env`, depending on the `ENVIRONMENT` value used in the `.env` file. -3. Create or update the `config//cron.d/service-crontab`, depending on the `ENVIRONMENT` value used in the `.env` file. This will defined the frequency with which the application will be executed. Note that you may need to create a new crontab file without the `.template` suffix shown in the template. -4. Build the image and start the service: +2. Create or update the `config//variables.env`, depending on the `ENVIRONMENT` value used in the `.env` file. Note that if the environment variable does not exist (which will happen if you have not copied it from the template), `docker` will complain. ``` -$ docker-compose up --build +$ docker compose up +ERROR: Couldn't find env file: /tmp//config/pre/variables.env +``` + +3. Build the image and start the service: + +``` +$ docker compose up ``` Consider using similar approaches for any other new services deployed in the `docker-compose.yml` file. @@ -22,7 +27,7 @@ Consider using similar approaches for any other new services deployed in the `do In serious projects, you will ship patches and features from time to time. Bringing this patches onto the project is easier now that production environment variables are shipped apart. -After shutting down the server (`docker-compose down`), you can now bring your patches: +After shutting down the server (`docker compose down`), you can now bring your patches: ``` $ git pull @@ -34,7 +39,74 @@ However, as both, data and environment variables are kept apart from the code, y This can be done as stated in the main [`./README.md`](../README.md): ``` -$ docker-compose up --build +$ docker compose up --build ``` Note that it is strongly recommended to have this in mind when shipping upgrades and bug fixes: devops teams might be expecting to work in this way when shipping upgrades, so keeping environment-things documented and separated is desired. + +## Additional Docker and Docker Compose Tips + +To stop the containers, you can always use `docker compose down` from the project home. +At the same time, to reraise without building, use `docker compose up`. +Note that this approach will show the output in the terminal. +However, running this in detached mode can be useful. + +``` +$ docker compose up -d +``` + +To grab the logs of a specifc container set up using `docker-compose` you can use a specific command: `docker compose logs ` as follows: + +``` +$ docker compose logs +Attaching to temperediron_main_service_1 +main_service_1 | [INFO] tempered-iron @ 2022-04-05 15:45:00,779 | Checking the public IP… +main_service_1 | [DEBUG] tempered-iron @ 2022-04-05 15:45:00,781 | Starting new HTTPS connection (1): ifconfig.me:443 +main_service_1 | [DEBUG] tempered-iron @ 2022-04-05 15:45:01,040 | https://ifconfig.me:443 "GET / HTTP/1.1" 200 14 +... +main_service_1 | [DEBUG] tempered-iron @ 2022-04-05 15:52:00,738 | Starting new HTTPS connection (1): ifconfig.me:443 +main_service_1 | [DEBUG] tempered-iron @ 2022-04-05 15:52:00,962 | https://ifconfig.me:443 "GET / HTTP/1.1" 200 14 +main_service_1 | [INFO] tempered-iron @ 2022-04-05 15:52:00,964 | My public IP is: XX.XX.XX.XX +main_service_1 | [INFO] tempered-iron @ 2022-04-05 15:52:00,964 | Closing the application… +``` + +With this approach, the application will end a return to the prompt. +However, if you prefer to _follow_ the logs, you can use the `-f` or `--follow` as shown with `docker compose logs --help` so as to let the application wait for new lines: + +``` +$ docker compose logs --help +View output from containers. + +Usage: logs [options] [SERVICE...] + +Options: + --no-color Produce monochrome output. + -f, --follow Follow log output. + -t, --timestamps Show timestamps. + --tail="all" Number of lines to show from the end of the logs + for each container. +$ docker compose logs -f +``` + +Note that for debugging purposes you might find useful to enter the container using a shell. +This can be done using `docker exec -it` command once you know the name of the container. +In the aformentioned case, you can see in that the ID of the container is `425bae76ea02` from the output. + +If you have already run the container, you can check the running ones using `docker ps` to get the ID of the container you want to enter. + +``` +$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +cf5d42a2da7d temperediron_main_service "sh -c 'crond -f' sh…" 2 minutes ago Up 2 minutes temperediron_main_service_1 +``` + +In this case, entering the container is as simple as entering the container with a shell in interactive mode (`-it`). +Note that the default shell of Alpine containers is `sh`, but use `bash` if it is available in the Docker image you used as a base image. + +``` +$ docker exec -it cf5d42a2da7d sh +/app # whoami +root +/app # ls +run.py +``` diff --git a/docs/README.md b/docs/README.md index 71e265c..ee9a7ff 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,6 +3,7 @@ You can find more information about the project in the following help files: - [`./ENVIRONMENT_VARIABLES.md`](./ENVIRONMENT_VARIABLES.md). More information about how to work with environment variables. +- [`./LESSONS_LEARNED.md`](./LESSONS_LEARNED.md). A list of concepts widely used in the repository which may be useful for those wanting to understand how things simply work out-of-the-box. - [`./PRODUCTION_RECOMMENDATIONS.md`](./PRODUCTION_RECOMMENDATIONS.md). More information about tips on how to move from `dev` to `pro`. - [`./VOLUME_MANAGEMENT.md`](./VOLUME_MANAGEMENT.md). More information about how volumes are used. diff --git a/docs/VOLUME_MANAGEMENT.md b/docs/VOLUME_MANAGEMENT.md index 7d738a9..c9af2b5 100644 --- a/docs/VOLUME_MANAGEMENT.md +++ b/docs/VOLUME_MANAGEMENT.md @@ -5,17 +5,17 @@ They can be used to share the contents of a folder in the host with the containe ## Conventional Approach -Typically, in `docker-compose` you can define volumes with absolute paths in the host. +Typically, in the `docker-compose.yml` you can define volumes with absolute paths in the host. For example, ``` ... volumes: - - ./data_vol:/data - - ./log_vol:/log + - ./volumes/data:/data + - ./volumes/log:/log ... ``` -This would map a local folder located at `./data_vol` (relative to the `docker-compose.yml` path) with a volume that will be mounted at `/data/`. +This would map a local folder located at `./volumes/data` (relative to the `docker-compose.yml` path) with a volume that will be mounted at `/data/`. Thus, the contents located at the folder can be accessed from both, the container and the host. ## Production ready @@ -53,7 +53,7 @@ services volumes: - data_vol:/data - log_vol:/log - - ./config/${ENVIRONMENT:-dev}/cron.d:/etc/cron.d + - ./config/${ENVIRONMENT:-dev}/crontabs:/etc/crontabs volumes: data_vol: @@ -73,7 +73,7 @@ services volumes: - data_vol:/data - log_vol:/log - - ./config/${ENVIRONMENT:-dev}/cron.d:/etc/cron.d + - ./config/${ENVIRONMENT:-dev}/crontabs:/etc/crontabs volumes: data_vol: name: pro_data_vol @@ -86,5 +86,5 @@ volumes: This is good because different environments can be used safely in a single machine without mixing others. Of course, additional volumes can be added depending on the needs. -The only exception as shown above is the example of the configuration of how the `/etc/cron.d/` is mapped from the host. -This information is grabbed from the `./config/${ENVIRONMENT}/cron.d` folder meaning that the scheduled tasks can be mapped at deployment time. \ No newline at end of file +The only exception as shown above is the example of the configuration of how the `/etc/crontabs/` is mapped from the host. +This information is grabbed from the `./config/${ENVIRONMENT}/crontabs` folder meaning that the scheduled tasks can be mapped at deployment time.