From decbb72fa667e2fc53c8078a10affc29c53e8b76 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 8 May 2024 14:28:46 -0500 Subject: [PATCH 1/3] Add tiler-imposm --- .github/workflows/chartpress.yaml | 315 +++-- .gitignore | 8 +- chartpress.yaml | 2 + compose/tiler.yml | 26 + images/tiler-imposm/Dockerfile | 61 + images/tiler-imposm/README.md | 31 + images/tiler-imposm/config/imposm3.json | 1134 +++++++++++++++++ .../tiler-imposm/config/postgis_helpers.sql | 19 + images/tiler-imposm/config/postgis_index.sql | 17 + images/tiler-imposm/liveness.sh | 8 + images/tiler-imposm/scripts/natural_earth.sh | 157 +++ images/tiler-imposm/scripts/osm_land.sh | 60 + images/tiler-imposm/start.sh | 174 +++ 13 files changed, 1847 insertions(+), 165 deletions(-) create mode 100644 images/tiler-imposm/Dockerfile create mode 100644 images/tiler-imposm/README.md create mode 100644 images/tiler-imposm/config/imposm3.json create mode 100644 images/tiler-imposm/config/postgis_helpers.sql create mode 100644 images/tiler-imposm/config/postgis_index.sql create mode 100755 images/tiler-imposm/liveness.sh create mode 100755 images/tiler-imposm/scripts/natural_earth.sh create mode 100755 images/tiler-imposm/scripts/osm_land.sh create mode 100755 images/tiler-imposm/start.sh diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index c24e9586..932400cd 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -4,168 +4,163 @@ on: branches: - 'main' - 'staging' - - 'ghcr' + - 'tiler-imposm' jobs: build: runs-on: ubuntu-20.04 timeout-minutes: 120 steps: - - uses: actions/checkout@v1 - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GHCR_GITHUB_TOKEN }} - - name: Setup python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Setup git - run: git config --global user.email "noreply@developmentseed.org" && git config --global user.name "Github Action" - - name: Install Chartpress - run: | - pip install chartpress six ruamel.yaml - - name: Run Chartpress - run: chartpress --push - env: - GITHUB_TOKEN: ${{ secrets.GHCR_GITHUB_TOKEN }} - - name: Staging - substitute secrets - if: github.ref == 'refs/heads/staging' - uses: bluwy/substitute-string-action@v1 - with: - _input-file: 'values.staging.template.yaml' - _format-key: '{{key}}' - _output-file: 'values.staging.yaml' - AWS_SSL_ARN: ${{ secrets.AWS_SSL_ARN }} - MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }} - MAILER_DOMAIN: ${{ secrets.MAILER_DOMAIN }} - MAILER_PASSWORD: ${{ secrets.STAGING_MAILER_PASSWORD }} - MAILER_USERNAME: ${{ secrets.STAGING_MAILER_USERNAME }} - STAGING_DB: ${{ secrets.STAGING_DB }} - STAGING_DB_EBS: ${{ secrets.STAGING_DB_EBS }} - STAGING_DB_PASSWORD: ${{ secrets.STAGING_DB_PASSWORD }} - STAGING_DB_USER: ${{ secrets.STAGING_DB_USER }} - STAGING_DOMAIN_NAME: staging.openhistoricalmap.org - STAGING_ID_KEY: ${{ secrets.STAGING_ID_KEY }} - STAGING_ID_APPLICATION: ${{ secrets.STAGING_ID_APPLICATION }} - STAGING_OAUTH_CLIENT_ID: ${{ secrets.STAGING_OAUTH_CLIENT_ID }} - STAGING_OAUTH_KEY: ${{ secrets.STAGING_OAUTH_KEY }} - STAGING_S3_BUCKET: ${{ secrets.STAGING_S3_BUCKET }} - # STAGING_TILER_DB: ${{ secrets.STAGING_TILER_DB }} - # STAGING_TILER_DB_EBS: ${{ secrets.STAGING_TILER_DB_EBS }} - # STAGING_TILER_DB_HOST: ${{ secrets.STAGING_TILER_DB_HOST }} - STAGING_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }} - # STAGING_TILER_DB_USER: ${{ secrets.STAGING_TILER_DB_USER }} - # STAGING_TILER_IMPOSM_EBS: ${{ secrets.STAGING_TILER_IMPOSM_EBS }} - STAGING_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.STAGING_TILER_IMPOSM_REPLICATION_URL }} - # STAGING_TILER_SERVER_EBS: ${{ secrets.STAGING_TILER_SERVER_EBS }} - STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }} - STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }} - STAGING_TM_API_CONSUMER_KEY: ${{ secrets.STAGING_TM_API_CONSUMER_KEY }} - STAGING_TM_API_CONSUMER_SECRET: ${{ secrets.STAGING_TM_API_CONSUMER_SECRET }} - STAGING_TM_API_DB: ${{ secrets.STAGING_TM_API_DB }} - STAGING_TM_API_DB_HOST: ${{ secrets.STAGING_TM_API_DB_HOST }} - STAGING_TM_API_DB_PASSWORD: ${{ secrets.STAGING_TM_API_DB_PASSWORD }} - STAGING_TM_API_DB_USER: ${{ secrets.STAGING_TM_API_DB_USER }} - STAGING_TM_API_SECRET: ${{ secrets.STAGING_TM_API_SECRET }} - # STAGING_NOMINATIM_DB_EBS: ${{ secrets.STAGING_NOMINATIM_DB_EBS }} - # STAGING_NOMINATIM_PG_PORT: ${{ secrets.STAGING_NOMINATIM_PG_PORT }} - # STAGING_NOMINATIM_PG_USER: ${{ secrets.STAGING_NOMINATIM_PG_USER }} - STAGING_NOMINATIM_PG_PASSWORD: ${{ secrets.STAGING_NOMINATIM_PG_PASSWORD }} - # STAGING_NOMINATIM_PG_DATABASE: ${{ secrets.STAGING_NOMINATIM_PG_DATABASE }} - # STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }} - STAGING_NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_NEW_RELIC_LICENSE_KEY }} - STAGING_NEW_RELIC_APP_NAME: ${{ secrets.STAGING_NEW_RELIC_APP_NAME }} - STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }} - STAGING_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }} - STAGING_OSMCHA_PG_DATABASE: ${{ secrets.STAGING_OSMCHA_PG_DATABASE }} - STAGING_OSMCHA_API_CONSUMER_KEY: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_KEY }} - STAGING_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_SECRET }} - STAGING_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }} - STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }} - - name: Production - substitute secrets - if: github.ref == 'refs/heads/main' - uses: bluwy/substitute-string-action@v1 - with: - _input-file: 'values.production.template.yaml' - _format-key: '{{key}}' - _output-file: 'values.production.yaml' - AWS_SSL_ARN: ${{ secrets.AWS_SSL_ARN }} - MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }} - MAILER_DOMAIN: ${{ secrets.MAILER_DOMAIN }} - MAILER_PASSWORD: ${{ secrets.MAILER_PASSWORD }} - MAILER_USERNAME: ${{ secrets.MAILER_USERNAME }} - PRODUCTION_DB: ${{ secrets.PRODUCTION_DB }} - PRODUCTION_DB_EBS: ${{ secrets.PRODUCTION_DB_EBS }} - PRODUCTION_DB_PASSWORD: ${{ secrets.PRODUCTION_DB_PASSWORD }} - PRODUCTION_DB_USER: ${{ secrets.PRODUCTION_DB_USER }} - PRODUCTION_DOMAIN_NAME: ${{ secrets.PRODUCTION_DOMAIN_NAME }} - PRODUCTION_ID_KEY: ${{ secrets.PRODUCTION_ID_KEY }} - PRODUCTION_ID_APPLICATION: ${{ secrets.PRODUCTION_ID_APPLICATION }} - PRODUCTION_OAUTH_CLIENT_ID: ${{ secrets.PRODUCTION_OAUTH_CLIENT_ID }} - PRODUCTION_OAUTH_KEY: ${{ secrets.PRODUCTION_OAUTH_KEY }} - PRODUCTION_S3_BUCKET: ${{ secrets.PRODUCTION_S3_BUCKET }} - PRODUCTION_DB_BACKUP_S3_BUCKET: ${{ secrets.PRODUCTION_DB_BACKUP_S3_BUCKET }} - PRODUCTION_TILER_DB: ${{ secrets.PRODUCTION_TILER_DB }} - PRODUCTION_TILER_DB_EBS: ${{ secrets.PRODUCTION_TILER_DB_EBS }} - PRODUCTION_TILER_DB_HOST: ${{ secrets.PRODUCTION_TILER_DB_HOST }} - PRODUCTION_TILER_DB_PASSWORD: ${{ secrets.PRODUCTION_TILER_DB_PASSWORD }} - PRODUCTION_TILER_DB_USER: ${{ secrets.PRODUCTION_TILER_DB_USER }} - PRODUCTION_TILER_IMPOSM_EBS: ${{ secrets.PRODUCTION_TILER_IMPOSM_EBS }} - PRODUCTION_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.PRODUCTION_TILER_IMPOSM_REPLICATION_URL }} - PRODUCTION_TILER_SERVER_EBS: ${{ secrets.PRODUCTION_TILER_SERVER_EBS }} - PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID }} - PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY }} - PRODUCTION_TM_API_CONSUMER_KEY: ${{ secrets.PRODUCTION_TM_API_CONSUMER_KEY }} - PRODUCTION_TM_API_CONSUMER_SECRET: ${{ secrets.PRODUCTION_TM_API_CONSUMER_SECRET }} - PRODUCTION_TM_API_DB: ${{ secrets.PRODUCTION_TM_API_DB }} - PRODUCTION_TM_API_DB_HOST: ${{ secrets.PRODUCTION_TM_API_DB_HOST }} - PRODUCTION_TM_API_DB_PASSWORD: ${{ secrets.PRODUCTION_TM_API_DB_PASSWORD }} - PRODUCTION_TM_API_DB_USER: ${{ secrets.PRODUCTION_TM_API_DB_USER }} - PRODUCTION_TM_API_SECRET: ${{ secrets.PRODUCTION_TM_API_SECRET }} - PRODUCTION_NOMINATIM_DB_EBS: ${{ secrets.PRODUCTION_NOMINATIM_DB_EBS }} - PRODUCTION_NOMINATIM_PG_PORT: ${{ secrets.PRODUCTION_NOMINATIM_PG_PORT }} - PRODUCTION_NOMINATIM_PG_USER: ${{ secrets.PRODUCTION_NOMINATIM_PG_USER }} - PRODUCTION_NOMINATIM_PG_PASSWORD: ${{ secrets.PRODUCTION_NOMINATIM_PG_PASSWORD }} - PRODUCTION_NOMINATIM_PG_DATABASE: ${{ secrets.PRODUCTION_NOMINATIM_PG_DATABASE }} - PRODUCTION_OVERPASS_API_DB_EBS: ${{ secrets.PRODUCTION_OVERPASS_API_DB_EBS }} - PRODUCTION_NEW_RELIC_LICENSE_KEY: ${{ secrets.PRODUCTION_NEW_RELIC_LICENSE_KEY }} - PRODUCTION_NEW_RELIC_APP_NAME: ${{ secrets.PRODUCTION_NEW_RELIC_APP_NAME }} - PRODUCTION_OSMCHA_PG_USER: ${{ secrets.PRODUCTION_OSMCHA_PG_USER }} - PRODUCTION_OSMCHA_PG_PASSWORD: ${{ secrets.PRODUCTION_OSMCHA_PG_PASSWORD }} - PRODUCTION_OSMCHA_PG_DATABASE: ${{ secrets.PRODUCTION_OSMCHA_PG_DATABASE }} - PRODUCTION_OSMCHA_API_CONSUMER_KEY: ${{ secrets.PRODUCTION_OSMCHA_API_CONSUMER_KEY }} - PRODUCTION_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.PRODUCTION_OSMCHA_API_CONSUMER_SECRET }} - PRODUCTION_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.PRODUCTION_OSMCHA_DJANGO_SECRET_KEY }} - PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }} - - name: AWS Credentials - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Setup Kubectl and Helm Dependencies - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n" - - name: Update kube-config staging - if: github.ref == 'refs/heads/staging' - run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging - - - name: Update kube-config prod - if: github.ref == 'refs/heads/main' - run: aws eks --region us-east-1 update-kubeconfig --name osmseed-production-v2 - - - name: Install helm dependencies for - if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: cd ohm && helm dep up - - - name: Staging - helm deploy - if: github.ref == 'refs/heads/staging' - run: helm upgrade --install staging --wait ohm/ -f values.staging.yaml -f ohm/values.yaml - - - name: Production - helm deploy - if: github.ref == 'refs/heads/main' - run: helm upgrade --install production --wait ohm/ -f values.production.yaml -f ohm/values.yaml + - uses: actions/checkout@v1 + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GHCR_GITHUB_TOKEN }} + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Setup git + run: git config --global user.email "noreply@developmentseed.org" && git config --global user.name "Github Action" + - name: Install Chartpress + run: | + pip install chartpress six ruamel.yaml + - name: Run Chartpress + run: chartpress --push + env: + GITHUB_TOKEN: ${{ secrets.GHCR_GITHUB_TOKEN }} + - name: Staging - substitute secrets + if: github.ref == 'refs/heads/staging' + uses: bluwy/substitute-string-action@v1 + with: + _input-file: 'values.staging.template.yaml' + _format-key: '{{key}}' + _output-file: 'values.staging.yaml' + AWS_SSL_ARN: ${{ secrets.AWS_SSL_ARN }} + MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }} + MAILER_DOMAIN: ${{ secrets.MAILER_DOMAIN }} + MAILER_PASSWORD: ${{ secrets.STAGING_MAILER_PASSWORD }} + MAILER_USERNAME: ${{ secrets.STAGING_MAILER_USERNAME }} + STAGING_DB: ${{ secrets.STAGING_DB }} + STAGING_DB_EBS: ${{ secrets.STAGING_DB_EBS }} + STAGING_DB_PASSWORD: ${{ secrets.STAGING_DB_PASSWORD }} + STAGING_DB_USER: ${{ secrets.STAGING_DB_USER }} + STAGING_DOMAIN_NAME: staging.openhistoricalmap.org + STAGING_ID_KEY: ${{ secrets.STAGING_ID_KEY }} + STAGING_ID_APPLICATION: ${{ secrets.STAGING_ID_APPLICATION }} + STAGING_OAUTH_CLIENT_ID: ${{ secrets.STAGING_OAUTH_CLIENT_ID }} + STAGING_OAUTH_KEY: ${{ secrets.STAGING_OAUTH_KEY }} + STAGING_S3_BUCKET: ${{ secrets.STAGING_S3_BUCKET }} + # STAGING_TILER_DB: ${{ secrets.STAGING_TILER_DB }} + # STAGING_TILER_DB_EBS: ${{ secrets.STAGING_TILER_DB_EBS }} + # STAGING_TILER_DB_HOST: ${{ secrets.STAGING_TILER_DB_HOST }} + STAGING_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }} + # STAGING_TILER_DB_USER: ${{ secrets.STAGING_TILER_DB_USER }} + # STAGING_TILER_IMPOSM_EBS: ${{ secrets.STAGING_TILER_IMPOSM_EBS }} + STAGING_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.STAGING_TILER_IMPOSM_REPLICATION_URL }} + # STAGING_TILER_SERVER_EBS: ${{ secrets.STAGING_TILER_SERVER_EBS }} + STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }} + STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }} + STAGING_TM_API_CONSUMER_KEY: ${{ secrets.STAGING_TM_API_CONSUMER_KEY }} + STAGING_TM_API_CONSUMER_SECRET: ${{ secrets.STAGING_TM_API_CONSUMER_SECRET }} + STAGING_TM_API_DB: ${{ secrets.STAGING_TM_API_DB }} + STAGING_TM_API_DB_HOST: ${{ secrets.STAGING_TM_API_DB_HOST }} + STAGING_TM_API_DB_PASSWORD: ${{ secrets.STAGING_TM_API_DB_PASSWORD }} + STAGING_TM_API_DB_USER: ${{ secrets.STAGING_TM_API_DB_USER }} + STAGING_TM_API_SECRET: ${{ secrets.STAGING_TM_API_SECRET }} + # STAGING_NOMINATIM_DB_EBS: ${{ secrets.STAGING_NOMINATIM_DB_EBS }} + # STAGING_NOMINATIM_PG_PORT: ${{ secrets.STAGING_NOMINATIM_PG_PORT }} + # STAGING_NOMINATIM_PG_USER: ${{ secrets.STAGING_NOMINATIM_PG_USER }} + STAGING_NOMINATIM_PG_PASSWORD: ${{ secrets.STAGING_NOMINATIM_PG_PASSWORD }} + # STAGING_NOMINATIM_PG_DATABASE: ${{ secrets.STAGING_NOMINATIM_PG_DATABASE }} + # STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }} + STAGING_NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_NEW_RELIC_LICENSE_KEY }} + STAGING_NEW_RELIC_APP_NAME: ${{ secrets.STAGING_NEW_RELIC_APP_NAME }} + STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }} + STAGING_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }} + STAGING_OSMCHA_PG_DATABASE: ${{ secrets.STAGING_OSMCHA_PG_DATABASE }} + STAGING_OSMCHA_API_CONSUMER_KEY: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_KEY }} + STAGING_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_SECRET }} + STAGING_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }} + STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }} + - name: Production - substitute secrets + if: github.ref == 'refs/heads/main' + uses: bluwy/substitute-string-action@v1 + with: + _input-file: 'values.production.template.yaml' + _format-key: '{{key}}' + _output-file: 'values.production.yaml' + AWS_SSL_ARN: ${{ secrets.AWS_SSL_ARN }} + MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }} + MAILER_DOMAIN: ${{ secrets.MAILER_DOMAIN }} + MAILER_PASSWORD: ${{ secrets.MAILER_PASSWORD }} + MAILER_USERNAME: ${{ secrets.MAILER_USERNAME }} + PRODUCTION_DB: ${{ secrets.PRODUCTION_DB }} + PRODUCTION_DB_EBS: ${{ secrets.PRODUCTION_DB_EBS }} + PRODUCTION_DB_PASSWORD: ${{ secrets.PRODUCTION_DB_PASSWORD }} + PRODUCTION_DB_USER: ${{ secrets.PRODUCTION_DB_USER }} + PRODUCTION_DOMAIN_NAME: ${{ secrets.PRODUCTION_DOMAIN_NAME }} + PRODUCTION_ID_KEY: ${{ secrets.PRODUCTION_ID_KEY }} + PRODUCTION_ID_APPLICATION: ${{ secrets.PRODUCTION_ID_APPLICATION }} + PRODUCTION_OAUTH_CLIENT_ID: ${{ secrets.PRODUCTION_OAUTH_CLIENT_ID }} + PRODUCTION_OAUTH_KEY: ${{ secrets.PRODUCTION_OAUTH_KEY }} + PRODUCTION_S3_BUCKET: ${{ secrets.PRODUCTION_S3_BUCKET }} + PRODUCTION_DB_BACKUP_S3_BUCKET: ${{ secrets.PRODUCTION_DB_BACKUP_S3_BUCKET }} + PRODUCTION_TILER_DB: ${{ secrets.PRODUCTION_TILER_DB }} + PRODUCTION_TILER_DB_EBS: ${{ secrets.PRODUCTION_TILER_DB_EBS }} + PRODUCTION_TILER_DB_HOST: ${{ secrets.PRODUCTION_TILER_DB_HOST }} + PRODUCTION_TILER_DB_PASSWORD: ${{ secrets.PRODUCTION_TILER_DB_PASSWORD }} + PRODUCTION_TILER_DB_USER: ${{ secrets.PRODUCTION_TILER_DB_USER }} + PRODUCTION_TILER_IMPOSM_EBS: ${{ secrets.PRODUCTION_TILER_IMPOSM_EBS }} + PRODUCTION_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.PRODUCTION_TILER_IMPOSM_REPLICATION_URL }} + PRODUCTION_TILER_SERVER_EBS: ${{ secrets.PRODUCTION_TILER_SERVER_EBS }} + PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID }} + PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY }} + PRODUCTION_TM_API_CONSUMER_KEY: ${{ secrets.PRODUCTION_TM_API_CONSUMER_KEY }} + PRODUCTION_TM_API_CONSUMER_SECRET: ${{ secrets.PRODUCTION_TM_API_CONSUMER_SECRET }} + PRODUCTION_TM_API_DB: ${{ secrets.PRODUCTION_TM_API_DB }} + PRODUCTION_TM_API_DB_HOST: ${{ secrets.PRODUCTION_TM_API_DB_HOST }} + PRODUCTION_TM_API_DB_PASSWORD: ${{ secrets.PRODUCTION_TM_API_DB_PASSWORD }} + PRODUCTION_TM_API_DB_USER: ${{ secrets.PRODUCTION_TM_API_DB_USER }} + PRODUCTION_TM_API_SECRET: ${{ secrets.PRODUCTION_TM_API_SECRET }} + PRODUCTION_NOMINATIM_DB_EBS: ${{ secrets.PRODUCTION_NOMINATIM_DB_EBS }} + PRODUCTION_NOMINATIM_PG_PORT: ${{ secrets.PRODUCTION_NOMINATIM_PG_PORT }} + PRODUCTION_NOMINATIM_PG_USER: ${{ secrets.PRODUCTION_NOMINATIM_PG_USER }} + PRODUCTION_NOMINATIM_PG_PASSWORD: ${{ secrets.PRODUCTION_NOMINATIM_PG_PASSWORD }} + PRODUCTION_NOMINATIM_PG_DATABASE: ${{ secrets.PRODUCTION_NOMINATIM_PG_DATABASE }} + PRODUCTION_OVERPASS_API_DB_EBS: ${{ secrets.PRODUCTION_OVERPASS_API_DB_EBS }} + PRODUCTION_NEW_RELIC_LICENSE_KEY: ${{ secrets.PRODUCTION_NEW_RELIC_LICENSE_KEY }} + PRODUCTION_NEW_RELIC_APP_NAME: ${{ secrets.PRODUCTION_NEW_RELIC_APP_NAME }} + PRODUCTION_OSMCHA_PG_USER: ${{ secrets.PRODUCTION_OSMCHA_PG_USER }} + PRODUCTION_OSMCHA_PG_PASSWORD: ${{ secrets.PRODUCTION_OSMCHA_PG_PASSWORD }} + PRODUCTION_OSMCHA_PG_DATABASE: ${{ secrets.PRODUCTION_OSMCHA_PG_DATABASE }} + PRODUCTION_OSMCHA_API_CONSUMER_KEY: ${{ secrets.PRODUCTION_OSMCHA_API_CONSUMER_KEY }} + PRODUCTION_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.PRODUCTION_OSMCHA_API_CONSUMER_SECRET }} + PRODUCTION_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.PRODUCTION_OSMCHA_DJANGO_SECRET_KEY }} + PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }} + - name: AWS Credentials + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Setup Kubectl and Helm Dependencies + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n" + - name: Update kube-config staging + if: github.ref == 'refs/heads/staging' + run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging + - name: Update kube-config prod + if: github.ref == 'refs/heads/main' + run: aws eks --region us-east-1 update-kubeconfig --name osmseed-production-v2 + - name: Install helm dependencies for + if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' + run: cd ohm && helm dep up + - name: Staging - helm deploy + if: github.ref == 'refs/heads/staging' + run: helm upgrade --install staging --wait ohm/ -f values.staging.yaml -f ohm/values.yaml + - name: Production - helm deploy + if: github.ref == 'refs/heads/main' + run: helm upgrade --install production --wait ohm/ -f values.production.yaml -f ohm/values.yaml diff --git a/.gitignore b/.gitignore index 847d2c97..d66af4ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,4 @@ .env-tiler -tiler-db-data -tiler-imposm-data-2 -tiler-imposm-data -tiler-imposm images/data/ data/ images/.env @@ -12,4 +8,6 @@ secrets tegola envs/.env.tiler envs/.env.web -config.toml \ No newline at end of file +config.toml +.DS_Store +*/.DS_Store \ No newline at end of file diff --git a/chartpress.yaml b/chartpress.yaml index 1b2d4f04..44c67cec 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -8,6 +8,8 @@ charts: valuesPath: osm-seed.db.image tiler-db: valuesPath: osm-seed.tilerDb.image + tiler-imposm: + valuesPath: osm-seed.tilerImposm.image tiler-server: valuesPath: osm-seed.tilerServer.image tasking-manager-api: diff --git a/compose/tiler.yml b/compose/tiler.yml index 335d4aa6..ce7ba650 100644 --- a/compose/tiler.yml +++ b/compose/tiler.yml @@ -1,5 +1,31 @@ version: '3.8' services: + db: + image: ohm-tiler-db:v1 + build: + context: ../images/tiler-db + dockerfile: Dockerfile + ports: + - "5433:5432" + volumes: + - ../data/tiler-db-data:/var/lib/postgresql/data + env_file: + - ../envs/.env.tiler + restart: always + imposm: + image: ohm-tiler-imposm:v1 + build: + context: ../images/tiler-imposm + dockerfile: Dockerfile + volumes: + - ../data/tiler-imposm-data:/mnt/data + - ../images/tiler-imposm:/app + command: + - sh + - -c + - "sleep 60 && ./start.sh" + env_file: + - ../envs/.env.tiler tiler: image: ohm-tiler-server:v1 build: diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile new file mode 100644 index 00000000..e99ccc28 --- /dev/null +++ b/images/tiler-imposm/Dockerfile @@ -0,0 +1,61 @@ +FROM osgeo/gdal:ubuntu-small-3.2.3 + +RUN apt-get -y update +RUN apt-get install -y \ + g++ \ + libboost-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libexpat1-dev \ + zlib1g-dev \ + libbz2-dev \ + libpq-dev \ + libgeos++-dev \ + libproj-dev \ + libleveldb-dev \ + libgeos-dev \ + libprotobuf-dev \ + git-core \ + curl \ + wget \ + unzip \ + software-properties-common && \ + rm -rf /var/lib/apt/lists/* + +# # Install python +RUN add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \ + rm -rf /var/lib/apt/lists/* && \ + python3 -m pip install pip --upgrade && \ + python3 -m pip install wheel + +# Install postgresql-client +RUN apt-get update && apt-get install -y postgresql-client && \ + rm -rf /var/lib/apt/lists/* + +# Install go +RUN wget -c https://dl.google.com/go/go1.10.linux-amd64.tar.gz -O - | tar -xz -C /usr/local +ENV PATH $PATH:/usr/local/go/bin + +WORKDIR /go +ENV GOPATH /go +RUN go get github.com/omniscale/imposm3 +WORKDIR $GOPATH/src/github.com/omniscale/imposm3 +RUN git checkout v0.11.1 +RUN go install github.com/omniscale/imposm3/cmd/imposm +ENV PATH $PATH:$GOPATH/bin + +# Install awscli and gsutil to get the pbf file +RUN pip3 install -U setuptools +RUN pip3 install awscli +RUN pip3 install gsutil + +ENV IMPOSMDATA /mnt/data +RUN mkdir -p $IMPOSMDATA && chmod 777 -R $IMPOSMDATA +WORKDIR $IMPOSMDATA +VOLUME $IMPOSMDATA + +WORKDIR /osm +COPY . . +CMD ./start.sh \ No newline at end of file diff --git a/images/tiler-imposm/README.md b/images/tiler-imposm/README.md new file mode 100644 index 00000000..14f557ef --- /dev/null +++ b/images/tiler-imposm/README.md @@ -0,0 +1,31 @@ +# Tiler imposm + +This container is responsible to import the replication PBF files from osm-seed or OSM planet dump into the `tiler-db` + +If we are running the container for the first time the container will import the [OSM Land](http://data.openstreetmapdata.com/land-polygons-split-3857.zip) and [Natural Earth dataset](http://nacis.org/initiatives/natural-earth) and [osm-land] files into the data bases. [Check more here](https://github.com/go-spatial/tegola-osm#import-the-osm-land-and-natural-earth-dataset-requires-gdal-natural-earth-can-be-skipped-if-youre-only-interested-in-osm). + + +### Configuration + +In order to run this container we need environment variables, these can be found in the following files👇: + +- [.env.tiler-db.example](./../../envs/.env.tiler-db.example) +- [.env.tiler-imposm.example](./../../envs/.env.tiler-imposm.example) + +**Note**: Rename the above files as `.env.tiler-db` and `.env.tiler-imposm` + +#### Running tiler-imposm container + +```sh + # Docker compose + docker-compose run tiler-imposm + + # Docker + docker run \ + --env-file ./envs/.env.tiler-db \ + --env-file ./envs/.env.tiler-imposm \ + -v ${PWD}/data/tiler-imposm-data:/mnt/data \ + --network osm-seed_default \ + -it osmseed-tiler-imposm:v1 +``` + diff --git a/images/tiler-imposm/config/imposm3.json b/images/tiler-imposm/config/imposm3.json new file mode 100644 index 00000000..9ba07998 --- /dev/null +++ b/images/tiler-imposm/config/imposm3.json @@ -0,0 +1,1134 @@ +{ + "tags": { + "load_all": true, + "exclude": [ + "created_by", + "source", + "source:datetime" + ] + }, + "generalized_tables": { + "water_areas_gen1": { + "source": "water_areas", + "sql_filter": "ST_Area(geometry)>50000.000000", + "tolerance": 50.0 + }, + "water_areas_gen0": { + "source": "water_areas_gen1", + "sql_filter": "ST_Area(geometry)>500000.000000", + "tolerance": 200.0 + }, + "transport_lines_gen0": { + "source": "transport_lines_gen1", + "sql_filter": null, + "tolerance": 200.0 + }, + "transport_lines_gen1": { + "source": "transport_lines", + "sql_filter": "type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link') OR class IN('railway', 'route')", + "tolerance": 50.0 + }, + "water_lines_gen0": { + "source": "water_lines_gen1", + "sql_filter": null, + "tolerance": 200 + }, + "water_lines_gen1": { + "source": "water_lines", + "sql_filter": null, + "tolerance": 50.0 + }, + "landuse_areas_gen1": { + "source": "landuse_areas", + "sql_filter": "ST_Area(geometry)>50000.000000", + "tolerance": 50.0 + }, + "landuse_areas_gen0": { + "source": "landuse_areas_gen1", + "sql_filter": "ST_Area(geometry)>500000.000000", + "tolerance": 200.0 + } + }, + "tables": { + "admin_areas": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "integer", + "name": "admin_level", + "key": "admin_level" + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "boolint", + "name": "has_label", + "key": false + } + ], + "type": "polygon", + "mapping": { + "boundary": [ + "administrative" + ] + } + }, + "place_points": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "integer", + "name": "population", + "key": "population" + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "point", + "mapping": { + "place": [ + "__any__" + ] + } + }, + "place_areas": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "integer", + "name": "population", + "key": "population" + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "polygon", + "mapping": { + "place": [ + "__any__" + ] + } + }, + "landuse_areas": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "pseudoarea", + "name": "area", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "polygon", + "mappings": { + "landuse": { + "mapping": { + "landuse": [ + "__any__" + ] + } + }, + "leisure": { + "mapping": { + "leisure": [ + "__any__" + ] + } + }, + "natural": { + "mapping": { + "natural": [ + "__any__" + ] + } + } + } + }, + "landuse_lines": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "linestring", + "filters": { + "exclude_tags": [ + ["area", "yes"], + ["natural", "coastline"], + ["boundary", "administrative"] + ] + }, + "mappings": { + "landuse": { + "mapping": { + "landuse": [ + "__any__" + ] + } + }, + "leisure": { + "mapping": { + "leisure": [ + "__any__" + ] + } + }, + "natural": { + "mapping": { + "natural": [ + "__any__" + ] + } + } + } + }, + "landuse_points": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "pseudoarea", + "name": "area", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "point", + "mappings": { + "landuse": { + "mapping": { + "landuse": [ + "__any__" + ] + } + }, + "leisure": { + "mapping": { + "leisure": [ + "__any__" + ] + } + }, + "natural": { + "mapping": { + "natural": [ + "__any__" + ] + } + } + } + }, + "water_areas": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "pseudoarea", + "name": "area", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "polygon", + "mapping": { + "waterway": [ + "__any__" + ], + "landuse": [ + "basin", + "reservoir" + ], + "natural": [ + "water" + ] + } + }, + "water_lines": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "linestring", + "mapping": { + "waterway": [ + "__any__" + ], + "barrier": [ + "ditch" + ], + "natural": [ + "cliff" + ] + } + }, + "transport_points": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "string", + "name": "ref", + "key": "ref" + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "point", + "mappings": { + "railway": { + "mapping": { + "railway": [ + "__any__" + ] + } + }, + "highway": { + "mapping": { + "highway": [ + "__any__" + ] + } + }, + "aeroway": { + "mapping": { + "aeroway": [ + "__any__" + ] + } + }, + "route": { + "mapping": { + "route": [ + "__any__" + ] + } + } + } + }, + "transport_lines": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "boolint", + "name": "tunnel", + "key": "tunnel" + }, + { + "type": "boolint", + "name": "bridge", + "key": "bridge" + }, + { + "type": "direction", + "name": "oneway", + "key": "oneway" + }, + { + "type": "string", + "name": "ref", + "key": "ref" + }, + { + "type": "wayzorder", + "name": "z_order", + "key": "layer" + }, + { + "type": "string", + "name": "access", + "key": "access" + }, + { + "type": "string", + "name": "service", + "key": "service" + }, + { + "type": "string", + "name": "ford", + "key": "ford" + }, + { + "type": "mapping_key", + "name": "class", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "linestring", + "filters": { + "exclude_tags": [ + ["area", "yes"] + ] + }, + "mappings": { + "railway": { + "mapping": { + "railway": [ + "__any__" + ] + } + }, + "highway": { + "mapping": { + "highway": [ + "__any__" + ] + } + }, + "aeroway": { + "mapping": { + "aeroway": [ + "__any__" + ] + } + }, + "route": { + "mapping": { + "route": [ + "__any__" + ] + } + } + } + + }, + "transport_areas": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "string", + "name": "area", + "key": "area" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "polygon", + "mappings": { + "rail": { + "mapping": { + "railway": [ + "__any__" + ] + } + }, + "highway": { + "mapping": { + "highway": [ + "__any__" + ] + } + }, + "aeroway": { + "mapping": { + "aeroway": [ + "__any__" + ] + } + }, + "route": { + "mapping": { + "route": [ + "__any__" + ] + } + } + } + }, + "amenity_points": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "point", + "mapping": { + "amenity": [ + "__any__" + ] + } + }, + "amenity_areas": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "polygon", + "mapping": { + "amenity": [ + "__any__" + ] + } + }, + "other_points": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "point", + "mappings": { + "barrier": { + "mapping": { + "barrier": [ + "__any__" + ] + } + }, + "historic": { + "mapping": { + "historic": [ + "__any__" + ] + } + }, + "man_made": { + "mapping": { + "man_made": [ + "__any__" + ] + } + }, + "power": { + "mapping": { + "power": [ + "__any__" + ] + } + }, + "military": { + "mapping": { + "military": [ + "__any__" + ] + } + }, + "tourism": { + "mapping": { + "tourism": [ + "__any__" + ] + } + }, + "shop": { + "mapping": { + "shop": [ + "__any__" + ] + } + }, + "craft": { + "mapping": { + "craft": [ + "__any__" + ] + } + } + } + }, + "other_lines": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "linestring", + "mappings": { + "barrier": { + "mapping": { + "barrier": [ + "__any__" + ] + } + }, + "historic": { + "mapping": { + "historic": [ + "__any__" + ] + } + }, + "man_made": { + "mapping": { + "man_made": [ + "__any__" + ] + } + }, + "power": { + "mapping": { + "power": [ + "__any__" + ] + } + }, + "military": { + "mapping": { + "military": [ + "__any__" + ] + } + } + } + }, + "other_areas": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "pseudoarea", + "name": "area", + "key": null + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + } + ], + "type": "polygon", + "mappings": { + "barrier": { + "mapping": { + "barrier": [ + "__any__" + ] + } + }, + "historic": { + "mapping": { + "historic": [ + "__any__" + ] + } + }, + "man_made": { + "mapping": { + "man_made": [ + "__any__" + ] + } + }, + "power": { + "mapping": { + "power": [ + "__any__" + ] + } + }, + "military": { + "mapping": { + "military": [ + "__any__" + ] + } + } + } + }, + "buildings": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type":"string", + "name":"height", + "key": "height" + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "polygon", + "mapping": { + "building": [ + "__any__" + ] + } + }, + "relation_members": { + "type": "relation_member", + "fields": [ + { + "name": "osm_id", + "type": "id" + }, + { + "name": "member", + "type": "member_id" + }, + { + "name": "index", + "type": "member_index" + }, + { + "name": "role", + "type": "member_role" + }, + { + "name": "type", + "type": "member_type" + }, + { + "name": "geometry", + "type": "geometry" + }, + { + "name": "relname", + "key": "name", + "type": "string" + }, + { + "name": "name", + "key": "name", + "type": "string", + "from_member": true + } + ], + "mapping": { + "__any__": [ + "__any__" + ] + } + }, + "relations": { + "type": "relation", + "fields": [ + { + "name": "osm_id", + "type": "id" + }, + { + "key": "type", + "name": "type", + "type": "string" + }, + { + "key": "name", + "name": "name", + "type": "string" + }, + { + "name": "tags", + "type": "hstore_tags" + } + ], + "mapping": { + "__any__": [ + "__any__" + ] + } + } + } +} diff --git a/images/tiler-imposm/config/postgis_helpers.sql b/images/tiler-imposm/config/postgis_helpers.sql new file mode 100644 index 00000000..41f9d85d --- /dev/null +++ b/images/tiler-imposm/config/postgis_helpers.sql @@ -0,0 +1,19 @@ +/* + helper functions that should be installed alongside the OSM import +*/ + +BEGIN; + + -- Inspired by http://stackoverflow.com/questions/16195986/isnumeric-with-postgresql/16206123#16206123 +CREATE OR REPLACE FUNCTION as_numeric(text) RETURNS NUMERIC AS $$ +DECLARE test NUMERIC; +BEGIN + test = $1::NUMERIC; + RETURN test; +EXCEPTION WHEN others THEN + RETURN -1; +END; +$$ STRICT +LANGUAGE plpgsql IMMUTABLE; + +COMMIT; \ No newline at end of file diff --git a/images/tiler-imposm/config/postgis_index.sql b/images/tiler-imposm/config/postgis_index.sql new file mode 100644 index 00000000..9edf676b --- /dev/null +++ b/images/tiler-imposm/config/postgis_index.sql @@ -0,0 +1,17 @@ +/* + Adds indexes to OSM table columns to increase query performance per the tegola configuration. +*/ + +BEGIN; + CREATE INDEX ON osm_transport_lines_gen0 (type); + CREATE INDEX ON osm_transport_lines_gen1 (type); + CREATE INDEX ON osm_transport_lines (type); + CREATE INDEX ON osm_admin_areas (admin_level); + CREATE INDEX ON osm_landuse_areas_gen0 (type); + CREATE INDEX ON osm_water_lines (type); + CREATE INDEX ON osm_water_lines_gen0 (type); + CREATE INDEX ON osm_water_lines_gen1 (type); + CREATE INDEX ON osm_water_areas (type); + CREATE INDEX ON osm_water_areas_gen0 (type); + CREATE INDEX ON osm_water_areas_gen1 (type); +COMMIT; diff --git a/images/tiler-imposm/liveness.sh b/images/tiler-imposm/liveness.sh new file mode 100755 index 00000000..a7b249ea --- /dev/null +++ b/images/tiler-imposm/liveness.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +if ps aux | grep -v grep | grep "imposm" >/dev/null; then + echo "imposm process is running." + exit 0 +else + echo "imposm process is not running." 1>&2 + exit 1 +fi diff --git a/images/tiler-imposm/scripts/natural_earth.sh b/images/tiler-imposm/scripts/natural_earth.sh new file mode 100755 index 00000000..4a249c30 --- /dev/null +++ b/images/tiler-imposm/scripts/natural_earth.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +# This script will install natural earth data (http://www.naturalearthdata.com/downloads/) into a PostGIS database named DB_NAME. +# The script assumes the following utilities are installed: +# - psql: PostgreSQL client +# - ogr2ogr: GDAL vector lib +# - unzip: decompression util +# +# Usage +# Set the database connection variables, then run +# +# ./natural_earth.sh +# +# Important +# - This script is idempotent and will DROP the natural earth database if it already exists +# - In order for this script to work the DB_USER must have access to the 'postgres' database to create a new database + + +set -e + + + +CONFIG_FILE='' +DROP_DB=false + +while getopts ":c:dv" flag; do + case ${flag} in + c) + if [[ ! -r $OPTARG ]]; then echo "Config File $OPTARG Not Found!"; exit 2; + else echo "Using config file: $OPTARG"; CONFIG_FILE=$OPTARG + fi ;; + v) + echo "Running in Verbose Mode" + set -x ;; + d) + echo "Dropping Existing DB"; DROP_DB=true ;; + \?) + printf '\nUnrecognized option: -%s \nUsage: \n[-c file] Path to Config File \n[-d] drop existing database \n[-v] verbose\n' $OPTARG; exit 2 ;; + :) + echo "Option -$OPTARG requires an argument"; exit 2 ;; + esac +done + +# database connection variables +DB_NAME=$POSTGRES_DB +DB_HOST=$POSTGRES_HOST +DB_PORT=$POSTGRES_PORT +DB_USER=$POSTGRES_USER +DB_PW=$POSTGRES_PASSWORD + +# Check if we're using a config file +if [[ -r $CONFIG_FILE ]]; then source $CONFIG_FILE +elif [ -r dbcredentials.sh ]; then source dbcredentials.sh +fi + +# check our connection string before we do any downloading +psql "dbname='postgres' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" -c "\q" + +# array of natural earth dataset URLs + dataurls=( + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_admin_0_boundary_lines_land.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_admin_0_countries.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_admin_1_states_provinces_lines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_populated_places.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_coastline.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_geography_marine_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_geography_regions_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_rivers_lake_centerlines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_lakes.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_glaciated_areas.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_land.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_110m_ocean.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_admin_0_boundary_lines_land.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_admin_0_boundary_lines_disputed_areas.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_admin_0_boundary_lines_maritime_indicator.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_admin_0_countries.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_admin_0_map_subunits.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_admin_1_states_provinces_lakes.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_admin_1_states_provinces_lines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_populated_places.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_geographic_lines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_coastline.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_antarctic_ice_shelves_lines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_antarctic_ice_shelves_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_geography_marine_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_geography_regions_elevation_points.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_geography_regions_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_rivers_lake_centerlines_scale_rank.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_rivers_lake_centerlines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_lakes.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_glaciated_areas.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_land.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_50m_ocean.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_0_boundary_lines_land.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_0_boundary_lines_disputed_areas.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_parks_and_protected_lands.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_0_boundary_lines_map_units.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_0_boundary_lines_maritime_indicator.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_0_label_points.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_0_countries.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_0_map_subunits.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_1_label_points.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_admin_1_states_provinces_lines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_populated_places.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_roads.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_urban_areas.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_geographic_lines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_coastline.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_antarctic_ice_shelves_lines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_antarctic_ice_shelves_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_geography_marine_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_geography_regions_elevation_points.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_geography_regions_points.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_geography_regions_polys.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_rivers_north_america.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_rivers_europe.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_rivers_lake_centerlines_scale_rank.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_rivers_lake_centerlines.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_playas.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_reefs.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_lakes_historic.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_lakes_north_america.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_lakes_europe.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_lakes.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_glaciated_areas.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_land.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_minor_islands.zip" + "https://osmseed-staging.s3.amazonaws.com/naciscdn/ne_10m_ocean.zip" +) + +# remove old database if -d flag is set and create a new one +if [[ "$DROP_DB" = true ]]; +then + psql "dbname='postgres' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" -c "DROP DATABASE IF EXISTS $DB_NAME" + psql "dbname='postgres' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" -c "CREATE DATABASE $DB_NAME" +fi + +# Create postgis extension if it doesn't exist +psql "dbname='$DB_NAME' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" -c "CREATE EXTENSION IF NOT EXISTS postgis" + +# iterate our dataurls +for i in "${!dataurls[@]}"; do + url=${dataurls[$i]} + + echo "fetching $url"; + curl $url > $i.zip; + unzip $i -d $i + + # support for archives with more than one shapefile + for f in $i/*.shp; do + # reproject data to webmercator (3857) and insert into our database + OGR_ENABLE_PARTIAL_REPROJECTION=true ogr2ogr -unsetFieldWidth -t_srs EPSG:3857 -nlt PROMOTE_TO_MULTI -f PostgreSQL PG:"dbname='$DB_NAME' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" $f + done + + # clean up + rm -rf $i/ $i.zip +done diff --git a/images/tiler-imposm/scripts/osm_land.sh b/images/tiler-imposm/scripts/osm_land.sh new file mode 100755 index 00000000..04c819c5 --- /dev/null +++ b/images/tiler-imposm/scripts/osm_land.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -e + +CONFIG_FILE='' +while getopts ":c:v" flag; do + case ${flag} in + c) + if [[ ! -r $OPTARG ]]; then echo "Config File $OPTARG Not Found!"; exit 2; + else echo "Using config file: $OPTARG"; CONFIG_FILE=$OPTARG + fi ;; + v) + echo "Running in Verbose Mode" + set -x ;; + \?) + printf '\nUnrecognized option: -%s \nUsage: \n[-c file] Path to Config File \n[-v] verbose\n' $OPTARG; exit 2 ;; + :) + echo "Option -$OPTARG requires an argument"; exit 2 ;; + esac +done + +# database connection variables +DB_NAME=$POSTGRES_DB +DB_HOST=$POSTGRES_HOST +DB_PORT=$POSTGRES_PORT +DB_USER=$POSTGRES_USER +DB_PW=$POSTGRES_PASSWORD + +# Check if we're using a config file +if [[ -r $CONFIG_FILE ]]; then source $CONFIG_FILE +elif [ -r dbcredentials.sh ]; then source dbcredentials.sh +fi + +# check our connection string before we do any downloading +psql "dbname='postgres' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" -c "\q" + +# array of natural earth dataset URLs + dataurls=( + "https://osmdata.openstreetmap.de/download/land-polygons-split-3857.zip" +) + +psql "dbname='postgres' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" -c "DROP TABLE IF EXISTS land_polygons" + +# iterate our dataurls +for i in "${!dataurls[@]}"; do + url=${dataurls[$i]} + + echo "fetching $url"; + curl $url > $i.zip; + unzip $i -d $i + + shape_file=$(find $i -type f -name "*.shp") + + echo $shape_file + + # reproject data to webmercator (3857) and insert into our database + OGR_ENABLE_PARTIAL_REPROJECTION=true ogr2ogr -overwrite -t_srs EPSG:3857 -nlt PROMOTE_TO_MULTI -f PostgreSQL PG:"dbname='$DB_NAME' host='$DB_HOST' port='$DB_PORT' user='$DB_USER' password='$DB_PW'" $shape_file + + # clean up + rm -rf $i/ $i.zip +done diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh new file mode 100755 index 00000000..dfffdc0c --- /dev/null +++ b/images/tiler-imposm/start.sh @@ -0,0 +1,174 @@ +#!/bin/bash +set -ex + +# directories to keep the imposm's cache for updating the db +WORKDIR=/mnt/data +CACHE_DIR=$WORKDIR/cachedir +DIFF_DIR=$WORKDIR/diff +IMPOSM3_EXPIRE_DIR=$WORKDIR/imposm3_expire_dir + +PBFFILE="${WORKDIR}/osm.pbf" +STATEFILE="state.txt" +LIMITFILE="limitFile.geojson" +# # Setting directory +# settingDir=/osm +# Folder to store the imposm expider files in s3 or gs +BUCKET_IMPOSM_FOLDER=imposm +INIT_FILE=/mnt/data/init_done + +mkdir -p "$CACHE_DIR" "$DIFF_DIR" "$IMPOSM3_EXPIRE_DIR" + +# Create config file to set variables for imposm +{ + echo "{" + echo "\"cachedir\": \"$CACHE_DIR\"," + echo "\"diffdir\": \"$DIFF_DIR\"," + echo "\"connection\": \"postgis://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB\"," + echo "\"mapping\": \"config/imposm3.json\"," + echo "\"replication_url\": \"$REPLICATION_URL\"" + echo "}" +} >"$WORKDIR/config.json" + +function getData() { + ### Get the PBF file from the cloud provider or public URL + if [ "$TILER_IMPORT_FROM" == "osm" ]; then + if [ ! -f "$PBFFILE" ]; then + echo "$PBFFILE does not exist, downloading..." + wget "$TILER_IMPORT_PBF_URL" -O "$PBFFILE" + fi + fi +} + +getFormattedDate() { + local file_path="$1" + if command -v stat >/dev/null 2>&1; then + local modification_date=$(stat -c %Y "$file_path") + if [ $? -eq 0 ]; then + local formatted_date=$(date -d "@$modification_date" "+%Y-%m-%d:%H:%M:%S") + echo "Created/Updated date of $file_path: $formatted_date" + else + echo "Error: Unable to get file modification date for file ${file_path}" + fi + else + echo "Error: 'stat' command not found. Unable to get file modification date, for file ${file_path}" + fi +} + +function uploadExpiredFiles() { + # Upload the expired files to the cloud provider + for file in $(find "$IMPOSM3_EXPIRE_DIR" -type f -cmin -1); do + bucketFile=${file#*"$WORKDIR"} + getFormattedDate "$file" + # UPLOAD_EXPIRED_FILES=true to upload the expired to cloud provider + if [ "$UPLOAD_EXPIRED_FILES" == "true" ]; then + echo "Uploading expired file ${file} to ${AWS_S3_BUCKET}" + + # AWS + if [ "$CLOUDPROVIDER" == "aws" ]; then + aws s3 cp "$file" "${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}${bucketFile}" --acl public-read + fi + + # Google Storage + if [ "$CLOUDPROVIDER" == "gcp" ]; then + gsutil cp -a public-read "$file" "${GCP_STORAGE_BUCKET}${BUCKET_IMPOSM_FOLDER}${bucketFile}" + fi + else + echo "Expired files were not uploaded because UPLOAD_EXPIRED_FILES=${UPLOAD_EXPIRED_FILES}" + fi + done +} + +function updateData() { + ### Update the DB with the new data form minute replication + if [ "$OVERWRITE_STATE" = "true" ]; then + rm $DIFF_DIR/last.state.txt + fi + + # Check if last.state.txt exists + if [ -f "$DIFF_DIR/last.state.txt" ]; then + echo "Exist... $DIFF_DIR/last.state.txt" + else + # Create last.state.txt file with REPLICATION_URL and SEQUENCE_NUMBER from env vars + echo "timestamp=0001-01-01T00\:00\:00Z + sequenceNumber=$SEQUENCE_NUMBER + replicationUrl=$REPLICATION_URL" >$DIFF_DIR/last.state.txt + fi + + # Check if the limit file exists + if [ -z "$TILER_IMPORT_LIMIT" ]; then + imposm run -config "$WORKDIR/config.json" -expiretiles-dir "$IMPOSM3_EXPIRE_DIR" & + else + imposm run -config "$WORKDIR/config.json" -limitto "$WORKDIR/$LIMITFILE" -expiretiles-dir "$IMPOSM3_EXPIRE_DIR" & + fi + + while true; do + echo "Upload expired files... $(date +%F_%H-%M-%S)" + uploadExpiredFiles + sleep 1m + done +} + +function importData() { + ### Import the PBF and Natural Earth files to the DB + echo "Execute the missing functions" + psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_helpers.sql + + if [ "$IMPORT_NATURAL_EARTH" = "true" ]; then + echo "Importing Natural Earth..." + ./scripts/natural_earth.sh + fi + + if [ "$IMPORT_OSM_LAND" = "true" ]; then + echo "Import OSM Land..." + ./scripts/osm_land.sh + fi + + echo "Import PBF file..." + if [ -z "$TILER_IMPORT_LIMIT" ]; then + imposm import \ + -config $WORKDIR/config.json \ + -read $PBFFILE \ + -write \ + -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR + else + wget $TILER_IMPORT_LIMIT -O $WORKDIR/$LIMITFILE + imposm import \ + -config $WORKDIR/config.json \ + -read $PBFFILE \ + -write \ + -diff -cachedir $CACHE_DIR -overwritecache -diffdir $DIFF_DIR \ + -limitto $WORKDIR/$LIMITFILE + fi + + imposm import \ + -config $WORKDIR/config.json \ + -deployproduction + + # These index will help speed up tegola tile generation + psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_index.sql + + touch $INIT_FILE + + # Update the DB + updateData +} + +echo "Connecting to $POSTGRES_HOST DB" +flag=true +while "$flag" = true; do + pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue + # Change flag to false to stop ping the DB + flag=false + echo "Check if $INIT_FILE exists" + if ([[ -f $INIT_FILE ]]); then + echo "Update the DB with osm data" + updateData + else + echo "Import PBF data to DB" + getData + if [ -f $PBFFILE ]; then + echo "Start importing the data" + importData + fi + fi +done From fdf8479a120802d018d8f191688f43a2ab83707f Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 8 May 2024 15:04:14 -0500 Subject: [PATCH 2/3] Update values --- ohm/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ohm/values.yaml b/ohm/values.yaml index 30a52d6f..ffb7460c 100644 --- a/ohm/values.yaml +++ b/ohm/values.yaml @@ -11,6 +11,10 @@ osm-seed: image: name: chartpress_replace_me tag: chartpress_replace_me + tilerImposm: + image: + name: chartpress_replace_me + tag: chartpress_replace_me tilerServer: image: name: chartpress_replace_me From c0d53fbeedae5519f41167b668f60e3d459860f4 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 8 May 2024 15:09:12 -0500 Subject: [PATCH 3/3] Update queries for tiler-imposm --- images/tiler-imposm/config/postgis_index.sql | 1 + .../config/postgis_post_import.sql | 18 ++++++++++++++++++ values.staging.template.yaml | 4 ++-- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 images/tiler-imposm/config/postgis_post_import.sql diff --git a/images/tiler-imposm/config/postgis_index.sql b/images/tiler-imposm/config/postgis_index.sql index 9edf676b..cea5c563 100644 --- a/images/tiler-imposm/config/postgis_index.sql +++ b/images/tiler-imposm/config/postgis_index.sql @@ -14,4 +14,5 @@ BEGIN; CREATE INDEX ON osm_water_areas (type); CREATE INDEX ON osm_water_areas_gen0 (type); CREATE INDEX ON osm_water_areas_gen1 (type); + CREATE INDEX ON osm_relation_members (osm_id); COMMIT; diff --git a/images/tiler-imposm/config/postgis_post_import.sql b/images/tiler-imposm/config/postgis_post_import.sql new file mode 100644 index 00000000..76890abc --- /dev/null +++ b/images/tiler-imposm/config/postgis_post_import.sql @@ -0,0 +1,18 @@ +-- Create a function that updates the osm_admin_areas if the polygon has a label in the relations +CREATE OR REPLACE FUNCTION update_admin_areas() +RETURNS void AS $$ +BEGIN + UPDATE osm_admin_areas + SET has_label = TRUE + WHERE osm_id IN ( + SELECT osm_id + FROM osm_relation_members + WHERE role = 'label' + ); +END; +$$ LANGUAGE plpgsql; + +-- Execute the function +SELECT update_admin_areas(); + + diff --git a/values.staging.template.yaml b/values.staging.template.yaml index d0d33906..d28c1040 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -369,7 +369,7 @@ osm-seed: mountPath: /var/lib/postgresql/data subPath: postgresql-d # In case cloudProvider: aws - AWS_ElasticBlockStore_volumeID : vol-03294367f2687c585 + AWS_ElasticBlockStore_volumeID : vol-08446263c77f14c78 AWS_ElasticBlockStore_size: 200Gi resources: enabled: false @@ -407,7 +407,7 @@ osm-seed: accessMode: ReadWriteOnce mountPath: /mnt/data # In case cloudProvider: aws - AWS_ElasticBlockStore_volumeID: vol-086f2e43e4fa71f00 + AWS_ElasticBlockStore_volumeID: vol-0851f0e0b08bf84b9 AWS_ElasticBlockStore_size: 100Gi resources: enabled: false