Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat/ui-docker-container-builds #7149

Open
wants to merge 27 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
435ca8e
adding basic docker container
gilesw Nov 29, 2024
5899a76
missing newrelic config
gilesw Nov 29, 2024
6e70747
json entrypoint recommendation
gilesw Nov 29, 2024
9e067ea
allow frontend to be configured separately to web
gilesw Nov 29, 2024
ecc70ca
add postgres, redis, haproxy lb containers for dev
gilesw Nov 29, 2024
184bf2d
Merge branch 'main' into feat/ui-docker-container-builds
amontenegro Dec 2, 2024
3c4e09a
Merge branch 'main' into feat/ui-docker-container-builds
amontenegro Dec 3, 2024
036c286
missing env files
gilesw Dec 4, 2024
c170a93
Merge branch 'feat/ui-docker-container-builds' of github.com:ORCID/OR…
gilesw Dec 4, 2024
58795ec
ro only perms as full write causes startup error
gilesw Dec 4, 2024
f8923a7
drop yaml markers in env files
gilesw Dec 5, 2024
d8dad88
feat: add haproxy router for database connections and use static ips …
gilesw Dec 5, 2024
b338281
Fix the code so the app can start with an empty identifer_type table
amontenegro Dec 5, 2024
4e27c0d
Merge branch 'feat/ui-docker-container-builds' of https://github.com/…
amontenegro Dec 5, 2024
97f89bc
Adding users
amontenegro Dec 5, 2024
a99373d
Client added
amontenegro Dec 5, 2024
5485d4d
migrating back to a defined bridged network
gilesw Dec 6, 2024
1ab8139
Merge branch 'feat/ui-docker-container-builds' of github.com:ORCID/OR…
gilesw Dec 6, 2024
991c872
adding orcid-haprouter to the builds
gilesw Dec 6, 2024
34085f5
adding in haprouter as default postgres connection
gilesw Dec 6, 2024
8ae0c47
cleaner overridable haprouter config
gilesw Dec 9, 2024
935a391
default.env is transfered via anisble not dev.env
gilesw Dec 9, 2024
4de984c
default.env is transfered via anisble not dev.env
gilesw Dec 9, 2024
37453e2
Update passphrase
amontenegro Dec 9, 2024
60a1eff
Merge branch 'feat/ui-docker-container-builds' of https://github.com/…
amontenegro Dec 9, 2024
7150de1
handle stale containers
gilesw Dec 30, 2024
50ae7f1
missing permissions and use env
gilesw Dec 30, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: add haproxy router for database connections and use static ips …
…and host file records to not require docker dns
  • Loading branch information
gilesw committed Dec 5, 2024
commit d8dad88e6f904907b4d436e16f34faab14157da7
10 changes: 10 additions & 0 deletions docker-compose-cleandb.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/usr/bin/env bash

# Use this to test and initdb script sql file changes
#
docker compose down --volumes postgres

volume_name=$(basename `pwd` | tr '[:upper:]' '[:lower:]')_postgres_data

docker volume rm $volume_name -f

82 changes: 69 additions & 13 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: '2'
services:
dependencies:
image: orcid/registry-dependencies:${TAG:-0.0.1}
Expand All @@ -7,9 +6,16 @@ services:
context: .
args:
tag_numeric: ${TAG:-0.0.1}
# stop dependencies from being started with a compose up
profiles:
- build
networks:
custom_network:
ipv4_address: 10.20.0.2
extra_hosts:
- "dependencies:10.20.0.2"
- "redis:10.20.0.3"
- "postgres:10.20.0.4"
- "haprouter:10.20.0.5"

redis:
image: orcid/registry/redis:7.2.5-alpine
Expand All @@ -19,8 +25,16 @@ services:
context: .
dockerfile: redis/Dockerfile
profiles:
- database
- db
- dev
networks:
custom_network:
ipv4_address: 10.20.0.3
extra_hosts:
- "dependencies:10.20.0.2"
- "redis:10.20.0.3"
- "postgres:10.20.0.4"
- "haprouter:10.20.0.5"

postgres:
image: postgres:13.13-alpine3.19
Expand All @@ -34,12 +48,55 @@ services:
ports:
- '5432:5432'
profiles:
- database
- db
- dev
networks:
custom_network:
ipv4_address: 10.20.0.4
extra_hosts:
- "dependencies:10.20.0.2"
- "redis:10.20.0.3"
- "postgres:10.20.0.4"
- "haprouter:10.20.0.5"

haprouter:
image: ${DOCKER_REG_PRIVATE}/orcid/registry/orcid-haprouter:${TAG:-0.0.1}
build:
context: .
dockerfile: orcid-haprouter/Dockerfile
extra_hosts:
- "nowhere:127.0.0.1"
- "dependencies:10.20.0.2"
- "redis:10.20.0.3"
- "postgres:10.20.0.4"
- "haprouter:10.20.0.5"
environment:
POSTGRES_READ_FQDN_A: postgres
POSTGRES_READ_FQDN_B: nowhere
POSTGRES_READ_FQDN_C: nowhere
POSTGRES_WRITE_FQDN_A: postgres
POSTGRES_WRITE_FQDN_B: nowhere
POSTGRES_WRITE_FQDN_C: nowhere
SOLR_READ_FQDN_A: solr
SOLR_READ_FQDN_B: nowhere
SOLR_READ_FQDN_C: nowhere
SOLR_WRITE_FQDN_A: solr
SOLR_WRITE_FQDN_B: nowhere.local
SOLR_WRITE_FQDN_C: nowhere.local
ports:
- 0.0.0.0:8888:1936 # stats
- 0.0.0.0:7432:7432 # solr read
- 0.0.0.0:7983:7983 # solr write
- 0.0.0.0:7432:7432 # postgres read
- 0.0.0.0:6432:6432 # postgres write
networks:
custom_network:
ipv4_address: 10.20.0.5
profiles:
- dev

lb:
image: ${DOCKER_REG_PRIVATE}/orcid/registry/orcid-lb:${TAG:-0.0.1}
# entrypoint: sleep infinity
build:
context: .
dockerfile: orcid-lb/Dockerfile
Expand All @@ -49,10 +106,8 @@ services:
profiles:
- dev

# orcid-angular project
frontend:
image: ${DOCKER_REG_PRIVATE}/orcid/registry/orcid-web-frontend-${FRONTEND_LABEL:-qa}:${FRONTEND_TAG:-0.0.1}
# entrypoint: sleep infinity
build:
context: .
dockerfile: 'FIXME: must build in the orcid-angular project first Dockerfile.build'
Expand All @@ -64,7 +119,6 @@ services:

web_proxy:
image: ${DOCKER_REG_PRIVATE}/orcid/registry/orcid-web-proxy:${TAG:-0.0.1}
# entrypoint: sleep infinity
build:
context: .
dockerfile: orcid-web-proxy/Dockerfile
Expand All @@ -81,7 +135,6 @@ services:

web:
image: ${DOCKER_REG_PRIVATE}/orcid/registry/orcid-web:${TAG:-0.0.1}
# entrypoint: sleep infinity
build:
cache_from:
- orcid/registry-dependencies:${TAG:-0.0.1}
Expand All @@ -90,23 +143,26 @@ services:
args:
tag_numeric: ${TAG:-0.0.1}
env_file:
# defaults and dev config for all apps
- default.env
- properties/default.orcid_core.env
- properties/default.misc.env
- properties/default.frontend.env
- properties/default.persistence.env
# defaults and dev config per app
- orcid-web/default.env
# config written out by our deployment system
- orcid-web/deployment.env
# anything secure that is non prod separated goes here
- ${DOCKER_DEV_ENV_FILE:-empty.env}
ports:
- 0.0.0.0:13100:8080
profiles:
- dev
- ui

networks:
custom_network:
driver: bridge
ipam:
config:
- subnet: 10.20.0.0/16

volumes:
postgres_data:
4 changes: 4 additions & 0 deletions orcid-haprouter/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
FROM haproxy:2.4.24-bullseye

COPY orcid-haprouter/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg

169 changes: 169 additions & 0 deletions orcid-haprouter/haproxy.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
resolvers docker
nameserver dns 127.0.0.11:53
parse-resolv-conf
accepted_payload_size 8192
hold valid 10s
hold other 30s
hold refused 30s
hold nx 30s
hold timeout 30s
hold obsolete 30s

# How many times to retry a query
resolve_retries 3

# How long to wait between retries when no valid response has been received
timeout retry 1s

# How long to wait for a successful resolution
timeout resolve 1s

global
stats timeout 30s
daemon
maxconn 6000
# Default SSL material locations
ca-base /etc/ssl/certs
crt-base /etc/ssl/private
# Default ciphers to use on SSL-enabled listening sockets.
# For more information, see ciphers(1SSL). This list is from:
# https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
ssl-default-bind-ciphers kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
ssl-default-bind-options no-sslv3
# Default ciphers to use on SSL-enabled listening sockets.
# For more information, see ciphers(1SSL). This list is from:
# https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
ssl-default-server-ciphers kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
ssl-default-server-options no-sslv3
nbproc 1

tune.ssl.default-dh-param 2048
ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
ssl-default-bind-options no-sslv3

defaults
log global
mode http
option httplog
option dontlognull
option log-separate-errors
timeout connect 5s
timeout client 100s
timeout server 100s
stats show-modules
stats show-legends

listen stats-1936
description haproute Loadbalancer
bind 0.0.0.0:1936
mode http
stats enable
stats uri /
stats hide-version
stats show-node

#####################################################################################

#
# Frontends
#

# Solr

frontend solr-read-7983
description Frontend for Solr
bind 0.0.0.0:7983
mode http
timeout client 300s
monitor-uri /haproxy-status
acl solrs_dead nbsrv(reg-solr-read) lt 1
http-request set-log-level silent
default_backend reg-solr-read

frontend solr-write-6983
description Frontend for Solr writes
bind 0.0.0.0:6983
mode http
timeout client 300s
monitor-uri /haproxy-status
acl solrs_dead nbsrv(reg-solr-write) lt 1
http-request set-log-level silent
default_backend reg-solr-write

# Postgres

frontend reg-postgres-read-7432
description Frontend for Postgres read
bind 0.0.0.0:7432
mode tcp
timeout client 70m
monitor-uri /haproxy-status
acl postgres_dead nbsrv(reg-postgres-read) lt 1
http-request set-log-level silent
default_backend reg-postgres-read

frontend reg-postgres-write-6432
description Frontend for Postgres writes
bind 0.0.0.0:6432
mode tcp
timeout client 70m
monitor-uri /haproxy-status
acl postgres_dead nbsrv(reg-postgres-write) lt 1
http-request set-log-level silent
default_backend reg-postgres-write

#
# Backends
#

# Solr

backend reg-solr-read
description backend for solr cluster
mode http
balance leastconn
option httpchk GET /solr/profile/admin/ping
option redispatch 2
http-check expect status 200
timeout server 300s
timeout check 20s
default-server check maxconn 500 inter 20s init-addr libc,last,none
server "${SOLR_READ_FQDN_A}-read-a" "${SOLR_READ_FQDN_A}":8983
server "${SOLR_READ_FQDN_B}-read-b" "${SOLR_READ_FQDN_B}":8983
server "${SOLR_READ_FQDN_C}-read-c" "${SOLR_READ_FQDN_C}":8983

backend reg-solr-write
description Solr master running in tomcat statically set
mode http
balance leastconn
option httpchk GET /solr/profile/admin/ping
option redispatch 2
http-check expect status 200
timeout server 300s
timeout check 20s
default-server check maxconn 500 inter 20s init-addr libc,last,none
server "${SOLR_WRITE_FQDN_A}-write-a" "${SOLR_WRITE_FQDN_A}":8983
server "${SOLR_WRITE_FQDN_B}-write-b" "${SOLR_WRITE_FQDN_B}":8983
server "${SOLR_WRITE_FQDN_C}-write-c" "${SOLR_WRITE_FQDN_C}":8983

# Postgres

backend reg-postgres-read
mode tcp
balance leastconn
option pgsql-check user pgc
timeout server 70m
default-server inter 5000 fastinter 2000 downinter 5000 rise 2 fall 3 port 5432 init-addr libc,last,none
server "${POSTGRES_READ_FQDN_A}-read-a" "${POSTGRES_READ_FQDN_A}":5432 check port 5432
server "${POSTGRES_READ_FQDN_B}-read-b" "${POSTGRES_READ_FQDN_B}":5432 check port 5432
server "${POSTGRES_READ_FQDN_C}-read-c" "${POSTGRES_READ_FQDN_C}":5432 check port 5432

backend reg-postgres-write
mode tcp
balance leastconn
option pgsql-check user pgc
timeout server 70m
default-server inter 5000 fastinter 2000 downinter 5000 rise 2 fall 3 port 5432 init-addr libc,last,none
server "${POSTGRES_WRITE_FQDN_A}-write-a" "${POSTGRES_WRITE_FQDN_A}":5432 check port 5432
server "${POSTGRES_WRITE_FQDN_B}-write-b" "${POSTGRES_WRITE_FQDN_B}":5432 check port 5432
server "${POSTGRES_WRITE_FQDN_C}-write-c" "${POSTGRES_WRITE_FQDN_C}":5432 check port 5432
Loading