add docker

.dockerignore

@@ -0,0 +1,26 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

Dockerfile

@@ -0,0 +1,49 @@
+FROM python:3.10-slim
+
+#  Set a working directory inside the container
+WORKDIR /app
+
+RUN apt-get clean
+
+RUN apt-get update \
+-o Acquire::http::No-Cache=true \
+-o Acquire::http::Pipeline-Depth=0 \
+-o Acquire::BrokenProxy=true \
+&& apt-get install -y --no-install-recommends \
+-o Acquire::http::No-Cache=true \
+-o Acquire::http::Pipeline-Depth=0 \
+-o Acquire::BrokenProxy=true \
+    wget \
+    tar \
+    unzip \
+    gcc \
+    make \
+    libpq-dev \
+    git \
+    postgresql-server-dev-all \
+    && rm -rf /var/lib/apt/lists/*  # Clean up to reduce the image size
+
+# Download Flyway CLI
+RUN wget https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/11.0.0/flyway-commandline-11.0.0-linux-x64.tar.gz && \
+    tar -xvf flyway-commandline-11.0.0-linux-x64.tar.gz && \
+    mv flyway-11.0.0 /flyway && \
+    ln -s /flyway/flyway /usr/local/bin/flyway && \
+    rm flyway-commandline-11.0.0-linux-x64.tar.gz
+
+
+# Copy application files
+COPY . /app
+
+
+# Install Python dependencies from requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir -r requirements-dev.txt
+
+# Expose ports for flask and locust
+EXPOSE 5000
+EXPOSE 8089
+
+# Add Python path environment variable
+ENV PYTHONPATH "${PYTHONPATH}:/app"
+
+CMD ["flask", "run"]

README.md

@@ -75,11 +75,13 @@ We are always trying to improve our documentation. If you have suggestions or ru
 
    - See [Database migrations](https://github.com/fecgov/openFEC#database-migration) for more information on installing and configuring flyway
 
-2. Set up your Node environment— learn how to do this with our [Javascript Ecosystem Guide](https://github.com/18F/dev-environment-standardization/blob/18f-pages/pages/languages/javascript.md).
+2. Optionally, you can use [Docker Desktop](https://docs.docker.com/desktop/) to setup Python, Postgres, Elastic Search, and Flyway.
 
-3. Set up your Python environment— learn how to do this with our [Python Ecosystem Guide](https://github.com/18F/dev-environment-standardization/blob/18f-pages/pages/languages/python.md).
+3. Set up your Node environment— learn how to do this with our [Javascript Ecosystem Guide](https://github.com/18F/dev-environment-standardization/blob/18f-pages/pages/languages/javascript.md).
 
-4. Clone this repository.
+4. Set up your Python environment— learn how to do this with our [Python Ecosystem Guide](https://github.com/18F/dev-environment-standardization/blob/18f-pages/pages/languages/python.md).
+
+5. Clone this repository.
 
 ### Project dependencies
 
@@ -338,6 +340,44 @@ To update the version-controlled test subset after rebuilding, run:
 invoke dump postgresql://:@/cfdm_test data/subset.dump
 ```
 
+## Docker Laptop Setup
+
+To run the application locally using Docker:
+
+1. Clone the repository
+2. Follow the instructions above to install JavaScript dependencies
+3. Set the required environment variables:
+
+```
+   export SQLA_CONN=<your dev database connection string>
+	export SQLA_SAMPLE_DB_CONN=postgresql://postgres:password@db:5432/cfdm_test
+	export SQLA_DOCKER_DB_CONN=postgresql://postgres:password@db:5432/cfdm_test
+	export SQLA_TEST_CONN=postgresql://postgres:password@db:5432/cfdm_unit_test
+```
+
+3. Build the image:
+
+```
+docker-compose build
+```
+
+4. Start all services:
+
+```
+docker-compose up -d openfec db elasticsearch
+```
+
+5. Create and populate your local development database:
+
+```
+docker-compose exec -it db createdb -U postgres cfdm_test
+docker-compose exec openfec invoke create_sample_db
+```
+
+6. Visit your local version of the site http://localhost:5000
+
+See the wiki for more information on configuring [Docker for local setup](https://github.com/fecgov/openFEC/wiki/Docker-Setup-for-Local-Development), running and tesing [Elastic Search](https://github.com/fecgov/openFEC/wiki/Docker-Setup-and-Test-Elasticsearch-Locally), [Redis, and Celery](https://github.com/fecgov/openFEC/wiki/Docker-Setup-and-Test-Redis-and-Celery-Locally) locally.
+
 ## Deployment (FEC team only)
 
 ### Deployment prerequisites

data/flyway.toml

@@ -0,0 +1,26 @@
+# More information on the parameters can be found here: https://documentation.red-gate.com/flyway/flyway-cli-and-api/configuration/parameters
+
+[environments.local]
+url = "jdbc:postgresql://db:5432/cfdm_test"
+user = "postgres"
+password = "password"
+# jarDirs = ["path/to/java/migrations"]
+# driver =
+# schemas =
+# connectRetries =
+# connectRetriesInterval =
+# initSql =
+# jdbcProperties =
+# resolvers =
+
+[flyway]
+environment = "local"
+locations = ["filesystem:/app/data/migrations"]
+
+# [environments.build]
+# url = "jdbc:sqlite::memory:"
+# user = "buildUser"
+# password = "buildPassword"
+
+# [flyway.check]
+# buildEnvironment = "build"

docker-compose.yml

@@ -0,0 +1,95 @@
+services:
+  openfec:
+    platform: linux/amd64
+    build:
+      context: .
+    ports:
+      - "5000:5000"
+      - "8089:8089"
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_PUBLIC_BUCKET=${AWS_PUBLIC_BUCKET}
+      - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}
+      - FEC_API_URL=http://localhost:5000
+      - FEC_CMS_URL=http://localhost:8000
+      - FEC_DOWNLOAD_API_KEY=${FEC_DOWNLOAD_API_KEY}
+      - FEC_WEB_API_KEY_PUBLIC=${FEC_WEB_API_KEY_PUBLIC}
+      - FLASK_APP=webservices.rest.py
+      - FLASK_DEBUG=1
+      - FLASK_RUN_HOST=0.0.0.0
+      - IS_USING_DOCKER=true
+      - SLACK_HOOK=${SLACK_HOOK}
+      - SQLA_CONN=${SQLA_CONN}
+      - SQLA_DOCKER_DB_CONN=${SQLA_DOCKER_DB_CONN}
+      - SQLA_SAMPLE_DB_CONN=${SQLA_SAMPLE_DB_CONN}
+      - SQLA_TEST_CONN=${SQLA_TEST_CONN}
+    volumes:
+      - .:/app
+
+  db:
+    image: postgres:13
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: localdb
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
+    platform: linux/amd64
+    environment:
+      - discovery.type=single-node
+      - http.cors.enabled=true
+      - http.cors.allow-origin=*
+      - http.cors.allow-headers=X-Requested-With, Content-Type, Content-Length, Authorization
+      - http.cors.allow-methods=OPTIONS, HEAD, GET, POST, PUT, DELETE
+      - discovery.type=single-node
+    volumes:
+      - esdata:/usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+
+  redis:
+    image: redis:6.2
+    command: redis-server --save 60 1 --loglevel warning
+    ports:
+      - "6379:6379"
+
+  celery-worker:
+    build:
+      context: .
+    command: celery -A webservices.tasks  worker --loglevel=info
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_PUBLIC_BUCKET=${AWS_PUBLIC_BUCKET}
+      - AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - IS_USING_DOCKER=true
+      - SQLA_DOCKER_DB_CONN=${SQLA_DOCKER_DB_CONN}
+    depends_on:
+      - redis
+    volumes:
+      - .:/app
+
+  celery-beat:
+    build:
+      context: .
+    command: celery -A webservices.tasks beat --loglevel=info
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - IS_USING_DOCKER=true
+    depends_on:
+      - redis
+    volumes:
+      - .:/app
+
+volumes:
+  pgdata:
+  esdata:

webservices/legal_docs/advisory_opinions.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 import logging
 import re
-from webservices.rest import db
+from webservices.common.models import db
 from webservices.utils import (
     create_es_client,
     DateTimeEncoder,

webservices/legal_docs/archived_murs.py

@@ -2,7 +2,7 @@
 from elasticsearch_dsl import Search
 import logging
 import re
-from webservices.rest import db
+from webservices.common.models import db
 from webservices.utils import (
     create_es_client,
     create_eregs_link,

webservices/legal_docs/current_cases.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from webservices.rest import db
+from webservices.common.models import db
 from webservices.utils import (
     extend,
     create_es_client,

webservices/rest.py

@@ -67,9 +67,15 @@
 
 def sqla_conn_string():
     sqla_conn_string = env.get_credential('SQLA_CONN')
+
+    if not sqla_conn_string and env.get_credential("IS_USING_DOCKER"):
+        print("Environment variable SQLA_CONN is empty; trying SQLA_DOCKER_DB_CONN")
+        sqla_conn_string = env.get_credential('SQLA_DOCKER_DB_CONN')
+
     if not sqla_conn_string:
         print("Environment variable SQLA_CONN is empty; running against " + "local `cfdm_test`")
         sqla_conn_string = 'postgresql://:@/cfdm_test'
+
     return sqla_conn_string
 
 

webservices/tasks/__init__.py

@@ -81,6 +81,8 @@ def redis_url():
         redis_url = redis_env.credentials.get("uri")
 
         return redis_url
+    elif env.get_credential("IS_USING_DOCKER"):
+        return "redis://redis:6379/0"
 
     return env.get_credential("FEC_REDIS_URL", "redis://localhost:6379/0")
 

webservices/utils.py

@@ -524,7 +524,10 @@ def create_es_client():
             )
         else:
             # create local elasticsearch client
-            url = "http://localhost:9200"
+            if env.get_credential("IS_USING_DOCKER"):
+                url = "http://elasticsearch:9200"
+            else:
+                url = "http://localhost:9200"
             es_client = Elasticsearch(
                 url,
                 timeout=30,