[Issue #10] Populate the search index from the opportunity tables

api/Makefile

@@ -100,7 +100,7 @@ start-debug:
 run-logs: start
 	docker-compose logs --follow --no-color $(APP_NAME)
 
-init: build init-db
+init: build init-db init-opensearch
 
 clean-volumes: ## Remove project docker volumes (which includes the DB state)
 	docker-compose down --volumes
@@ -179,6 +179,19 @@ create-erds: # Create ERD diagrams for our DB schema
 setup-postgres-db: ## Does any initial setup necessary for our local database to work
 	$(PY_RUN_CMD) setup-postgres-db
 
+##################################################
+# Opensearch
+##################################################
+
+init-opensearch: start-opensearch
+# TODO - in subsequent PRs, we'll add more to this command to setup the search index locally
+
+start-opensearch:
+	docker-compose up --detach opensearch-node
+	docker-compose up --detach opensearch-dashboards
+	./bin/wait-for-local-opensearch.sh
+
+
 
 ##################################################
 # Testing

api/bin/wait-for-local-opensearch.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# wait-for-local-opensearch
+
+set -e
+
+# Color formatting
+RED='\033[0;31m'
+NO_COLOR='\033[0m'
+
+MAX_WAIT_TIME=30 # seconds
+WAIT_TIME=0
+
+# Curl the healthcheck endpoint of the local opensearch
+# until it returns a success response
+until curl --output /dev/null --silent http://localhost:9200/_cluster/health;
+do
+  echo "waiting on OpenSearch to initialize..."
+  sleep 3
+
+  WAIT_TIME=$(($WAIT_TIME+3))
+  if [ $WAIT_TIME -gt $MAX_WAIT_TIME ]
+  then
+    echo -e "${RED}ERROR: OpenSearch appears to not be starting up, running \"docker logs opensearch-node\" to troubleshoot.${NO_COLOR}"
+    docker logs opensearch-node
+    exit 1
+  fi
+done
+
+echo "OpenSearch is ready after ~${WAIT_TIME} seconds"
+
+

api/docker-compose.yml

@@ -12,6 +12,41 @@ services:
     volumes:
       - grantsdbdata:/var/lib/postgresql/data
 
+  opensearch-node:
+    image: opensearchproject/opensearch:latest
+    container_name: opensearch-node
+    environment:
+      - cluster.name=opensearch-cluster # Name the cluster
+      - node.name=opensearch-node # Name the node that will run in this container
+      - discovery.type=single-node # Nodes to look for when discovering the cluster
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM
+      - DISABLE_INSTALL_DEMO_CONFIG=true # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
+      - DISABLE_SECURITY_PLUGIN=true # Disables Security plugin
+    ulimits:
+      memlock:
+        soft: -1 # Set memlock to unlimited (no soft or hard limit)
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
+        hard: 65536
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data # Creates volume called opensearch-data and mounts it to the container
+    ports:
+      - 9200:9200 # REST API
+      - 9600:9600 # Performance Analyzer
+
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:latest
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601 # Map host port 5601 to container port 5601
+    expose:
+      - "5601" # Expose port 5601 for web access to OpenSearch Dashboards
+    environment:
+      - 'OPENSEARCH_HOSTS=["http://opensearch-node:9200"]'
+      - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true # disables security dashboards plugin in OpenSearch Dashboards
+
   grants-api:
     build:
       context: .
@@ -28,6 +63,8 @@ services:
       - .:/api
     depends_on:
       - grants-db
+      - opensearch-node
 
 volumes:
   grantsdbdata:
+  opensearch-data:

api/local.env

@@ -59,6 +59,15 @@ DB_SSL_MODE=allow
 # could contain sensitive information.
 HIDE_SQL_PARAMETER_LOGS=TRUE
 
+############################
+# Opensearch Environment Variables
+############################
+
+OPENSEARCH_HOST=opensearch-node
+OPENSEARCH_PORT=9200
+OPENSEARCH_USE_SSL=FALSE
+OPENSEARCH_VERIFY_CERTS=FALSE
+
 ############################
 # AWS Defaults
 ############################

api/pyproject.toml

@@ -22,6 +22,7 @@ gunicorn = "^22.0.0"
 psycopg = { extras = ["binary"], version = "^3.1.10" }
 pydantic-settings = "^2.0.3"
 flask-cors = "^4.0.0"
+opensearch-py = "^2.5.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.9.1"
@@ -43,6 +44,12 @@ sadisplay = "0.4.9"
 ruff = "^0.4.0"
 debugpy = "^1.8.1"
 freezegun = "^1.5.0"
+# This isn't the latest version of types-requests
+# because otherwise it depends on urllib3 v2 but opensearch-py
+# needs urlib3 v1. This should be temporary as opensearch-py
+# has an unreleased change to switch to v2, so I'm guessing
+# in the next few weeks we can just make this the latest?
+types-requests = "2.31.0.1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

api/src/adapters/search/__init__.py

@@ -0,0 +1,4 @@
+from src.adapters.search.opensearch_client import SearchClient
+from src.adapters.search.opensearch_config import get_opensearch_config
+
+__all__ = ["SearchClient", "get_opensearch_config"]