From 7f9ef7cc0b8a86e66bd3153fbbb47d760cd0f7b0 Mon Sep 17 00:00:00 2001 From: Gunar Gessner Date: Fri, 24 Feb 2023 11:49:09 -0300 Subject: [PATCH] feat(pizza): sync down users from production (#1370) * break: sync from production * chore: remove deprecated file * chore: remove long time deprecated file this script was used to migrate the database from Vultr to AWS it had been committed Just Because we might need it sometime we never needed if after that so better to remove the clutter * fix: update staging sync to use new script --- .github/workflows/sync-staging-db.yml | 28 +- docker-compose.yml | 14 +- scripts/pullrequest/create.sh | 7 - scripts/seed-database/.dockerignore | 1 - scripts/seed-database/Dockerfile | 12 - scripts/seed-database/README.md | 5 + scripts/seed-database/container.sh | 26 + scripts/seed-database/container.sql | 34 ++ scripts/seed-database/package.json | 22 - scripts/seed-database/pnpm-lock.yaml | 524 ------------------ scripts/seed-database/seed-db.sh | 7 - scripts/seed-database/sync.sh | 21 + .../seed-database/upsert-production-flows.js | 259 --------- scripts/upsert-flows.sh | 24 - 14 files changed, 108 insertions(+), 876 deletions(-) delete mode 100644 scripts/seed-database/.dockerignore delete mode 100644 scripts/seed-database/Dockerfile create mode 100644 scripts/seed-database/README.md create mode 100755 scripts/seed-database/container.sh create mode 100644 scripts/seed-database/container.sql delete mode 100644 scripts/seed-database/package.json delete mode 100644 scripts/seed-database/pnpm-lock.yaml delete mode 100644 scripts/seed-database/seed-db.sh create mode 100755 scripts/seed-database/sync.sh delete mode 100644 scripts/seed-database/upsert-production-flows.js delete mode 100755 scripts/upsert-flows.sh diff --git a/.github/workflows/sync-staging-db.yml b/.github/workflows/sync-staging-db.yml index 4baa0ac016..0fe2ea5222 100644 --- a/.github/workflows/sync-staging-db.yml +++ b/.github/workflows/sync-staging-db.yml @@ -15,18 +15,24 @@ jobs: - name: Install pulumi uses: pulumi/setup-pulumi@v2 - run: | - echo "HASURA_GRAPHQL_ADMIN_SECRET=$(pulumi config get hasura-admin-secret --stack staging)" >> $GITHUB_ENV - working-directory: infrastructure/application + echo "STAGING_PG_URL=$(pulumi stack output --stack staging --show-secrets dbRootUrl)" >> $GITHUB_ENV + working-directory: infrastructure/data env: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} - - uses: actions/setup-node@v2 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 with: - node-version: ${{ env.NODE_VERSION }} - - run: npm install -g pnpm - working-directory: scripts/seed-database - - run: pnpm install - working-directory: scripts/seed-database - - run: node ./upsert-production-flows.js --overwrite --limit 1 - working-directory: scripts/seed-database + aws-access-key-id: ${{ secrets.PIZZA_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.PIZZA_AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + - name: Copy .env files from Staging S3 to the current working directory with AWS CLI + run: ./scripts/pull-secrets.sh + - name: Check if .env files exist + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: ".env, .env.staging, api.planx.uk/.env.test, hasura.planx.uk/.env.test" + fail: true + - run: docker run --rm -v "./scripts/seed-database:/app" --workdir="/app" postgis/postgis:12-3.0-alpine "./container.sh ${STAGING_PG_URL} ${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" env: - HASURA_GRAPHQL_URL: https://hasura.editor.planx.dev/v1/graphql + PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS: ${{ secrets.PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS }} diff --git a/docker-compose.yml b/docker-compose.yml index d6e64124bc..2ac9ccaab7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,20 +96,16 @@ services: retries: 3 seed-database: - build: - context: ./scripts/seed-database + image: postgis/postgis:12-3.0-alpine volumes: - - "./hasura.planx.uk/:/hasura" + - "./scripts/seed-database:/app" + working_dir: "/app" + entrypoint: sh + command: -c "./container.sh postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres/${PG_DATABASE} ${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" restart: "no" depends_on: hasura-proxy: condition: service_healthy - environment: - HASURA_GRAPHQL_ADMIN_SECRET: ${HASURA_GRAPHQL_ADMIN_SECRET} - HASURA_GRAPHQL_URL: http://hasura-proxy:7000/v1/graphql - HASURA_GRAPHQL_ENDPOINT: http://hasura-proxy:7000/ - PRODUCTION_GRAPHQL_URL: 'https://hasura.editor.planx.uk/v1/graphql' - api: restart: unless-stopped build: diff --git a/scripts/pullrequest/create.sh b/scripts/pullrequest/create.sh index 6fb425eb87..5e61434b84 100755 --- a/scripts/pullrequest/create.sh +++ b/scripts/pullrequest/create.sh @@ -21,9 +21,6 @@ echo \ apt-get update -y apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y -# install hasura cli -curl -L https://github.com/hasura/graphql-engine/raw/stable/cli/get.sh | bash - # set env for this shell set -o allexport source .env.pizza @@ -35,7 +32,3 @@ docker compose \ -f docker-compose.yml \ -f docker-compose.pizza.yml \ up --build --wait - -# insert hasura seeds -cd hasura.planx.uk -hasura seed apply --envfile .env diff --git a/scripts/seed-database/.dockerignore b/scripts/seed-database/.dockerignore deleted file mode 100644 index c2658d7d1b..0000000000 --- a/scripts/seed-database/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -node_modules/ diff --git a/scripts/seed-database/Dockerfile b/scripts/seed-database/Dockerfile deleted file mode 100644 index de52a5b616..0000000000 --- a/scripts/seed-database/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM node:16.13.1-alpine -RUN npm install -g hasura-cli - -WORKDIR /scripts/ -RUN npm install -g pnpm@7.8.0 -COPY pnpm-lock.yaml . - -RUN pnpm fetch --prod -ADD . . -RUN pnpm install -f --recursive --offline --prod -COPY . . -CMD ["sh", "./seed-db.sh"] \ No newline at end of file diff --git a/scripts/seed-database/README.md b/scripts/seed-database/README.md new file mode 100644 index 0000000000..0ea45a0065 --- /dev/null +++ b/scripts/seed-database/README.md @@ -0,0 +1,5 @@ +# seed-database + +This script uses a read-only database user to selectively sync data from a production database to a local development database. + +This is useful for having production-grade data both locally and on ephemeral pizza links (i.e. vultr servers that are spun up for each Pull Request). diff --git a/scripts/seed-database/container.sh b/scripts/seed-database/container.sh new file mode 100755 index 0000000000..224635fbd6 --- /dev/null +++ b/scripts/seed-database/container.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# This is the script that runs inside the container +# Usage: container.sh + +# cd to this script's directory +cd "$(dirname "$0")" || exit + +set -ex + +LOCAL_PG="$1" +REMOTE_PG="$2" + +# fetch users +psql --command="\\COPY (SELECT * FROM users) TO '/tmp/users.csv' (FORMAT CSV, DELIMITER ';')" "${REMOTE_PG}" + +# fetch teams +psql --command="\\COPY (SELECT id, name, slug, theme, settings, domain FROM teams) TO '/tmp/teams.csv' (FORMAT CSV, DELIMITER ';')" "${REMOTE_PG}" + +# fetch flows +psql --command="\\COPY (SELECT * FROM flows) TO '/tmp/flows.csv' (FORMAT CSV, DELIMITER ';')" "${REMOTE_PG}" + +# fetch published_flows (the last two) +psql --command="\\COPY (SELECT id, data, flow_id, summary, publisher_id FROM (SELECT id, data, flow_id, summary, publisher_id, ROW_NUMBER() OVER (PARTITION BY flow_id ORDER BY created_at DESC) as row_num FROM published_flows) as subquery WHERE row_num <= 2) TO '/tmp/published_flows.csv' (FORMAT CSV);" "${REMOTE_PG}" + +# run container.sql +psql "${LOCAL_PG}" < container.sql diff --git a/scripts/seed-database/container.sql b/scripts/seed-database/container.sql new file mode 100644 index 0000000000..57f42b8b52 --- /dev/null +++ b/scripts/seed-database/container.sql @@ -0,0 +1,34 @@ +BEGIN; + +TRUNCATE TABLE users, teams CASCADE; + +\COPY users FROM '/tmp/users.csv' (FORMAT CSV, DELIMITER ';'); + +\COPY teams (id, name, slug, theme, settings, domain) FROM '/tmp/teams.csv' (FORMAT CSV, DELIMITER ';') + +\COPY flows FROM '/tmp/flows.csv' (FORMAT CSV, DELIMITER ';'); +UPDATE flows SET version = 1; + +-- insert an operation for each flow (to make sharedb happy) +INSERT INTO operations (flow_id, data, version) + SELECT + id + , json_build_object( + 'm', json_build_object( + 'ts', extract(epoch from now()) * 1000 + , 'uId', '1' + ) + , 'v', 0 + , 'seq', 1 + , 'src', '1' + , 'create', json_build_object( + 'data', '{}' + , 'type', 'http://sharejs.org/types/JSONv0' + ) + ) + , 1 + FROM flows; + +\COPY published_flows (id, data, flow_id, summary, publisher_id) FROM '/tmp/published_flows.csv' (FORMAT CSV); + +COMMIT; diff --git a/scripts/seed-database/package.json b/scripts/seed-database/package.json deleted file mode 100644 index 1e9d394f61..0000000000 --- a/scripts/seed-database/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "seed-database", - "private": true, - "license": "MPL-2.0", - "type": "module", - "scripts": { - "upsert-flows": "node --experimental-specifier-resolution=node ./upsert-production-flows" - }, - "engines": { - "node": "^16", - "pnpm": "^7.8.0" - }, - "dependencies": { - "graphql": "^16.5.0", - "graphql-request": "^4.3.0", - "meow": "^11.0.0", - "p-limit": "^4.0.0" - }, - "devDependencies": { - "@types/lodash": "^4.14.184" - } -} \ No newline at end of file diff --git a/scripts/seed-database/pnpm-lock.yaml b/scripts/seed-database/pnpm-lock.yaml deleted file mode 100644 index 4664f85839..0000000000 --- a/scripts/seed-database/pnpm-lock.yaml +++ /dev/null @@ -1,524 +0,0 @@ -lockfileVersion: 5.4 - -specifiers: - '@types/lodash': ^4.14.184 - graphql: ^16.5.0 - graphql-request: ^4.3.0 - meow: ^11.0.0 - p-limit: ^4.0.0 - -dependencies: - graphql: 16.6.0 - graphql-request: 4.3.0_graphql@16.6.0 - meow: 11.0.0 - p-limit: 4.0.0 - -devDependencies: - '@types/lodash': 4.14.184 - -packages: - - /@babel/code-frame/7.18.6: - resolution: {integrity: sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/highlight': 7.18.6 - dev: false - - /@babel/helper-validator-identifier/7.19.1: - resolution: {integrity: sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==} - engines: {node: '>=6.9.0'} - dev: false - - /@babel/highlight/7.18.6: - resolution: {integrity: sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/helper-validator-identifier': 7.19.1 - chalk: 2.4.2 - js-tokens: 4.0.0 - dev: false - - /@types/lodash/4.14.184: - resolution: {integrity: sha512-RoZphVtHbxPZizt4IcILciSWiC6dcn+eZ8oX9IWEYfDMcocdd42f7NPI6fQj+6zI8y4E0L7gu2pcZKLGTRaV9Q==} - dev: true - - /@types/minimist/1.2.2: - resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} - dev: false - - /@types/normalize-package-data/2.4.1: - resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} - dev: false - - /ansi-styles/3.2.1: - resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} - engines: {node: '>=4'} - dependencies: - color-convert: 1.9.3 - dev: false - - /arrify/1.0.1: - resolution: {integrity: sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==} - engines: {node: '>=0.10.0'} - dev: false - - /asynckit/0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: false - - /camelcase-keys/8.0.2: - resolution: {integrity: sha512-qMKdlOfsjlezMqxkUGGMaWWs17i2HoL15tM+wtx8ld4nLrUwU58TFdvyGOz/piNP842KeO8yXvggVQSdQ828NA==} - engines: {node: '>=14.16'} - dependencies: - camelcase: 7.0.0 - map-obj: 4.3.0 - quick-lru: 6.1.1 - type-fest: 2.19.0 - dev: false - - /camelcase/7.0.0: - resolution: {integrity: sha512-JToIvOmz6nhGsUhAYScbo2d6Py5wojjNfoxoc2mEVLUdJ70gJK2gnd+ABY1Tc3sVMyK7QDPtN0T/XdlCQWITyQ==} - engines: {node: '>=14.16'} - dev: false - - /chalk/2.4.2: - resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} - engines: {node: '>=4'} - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - dev: false - - /color-convert/1.9.3: - resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} - dependencies: - color-name: 1.1.3 - dev: false - - /color-name/1.1.3: - resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - dev: false - - /combined-stream/1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - dependencies: - delayed-stream: 1.0.0 - dev: false - - /cross-fetch/3.1.5: - resolution: {integrity: sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==} - dependencies: - node-fetch: 2.6.7 - transitivePeerDependencies: - - encoding - dev: false - - /decamelize-keys/1.1.1: - resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} - engines: {node: '>=0.10.0'} - dependencies: - decamelize: 1.2.0 - map-obj: 1.0.1 - dev: false - - /decamelize/1.2.0: - resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} - engines: {node: '>=0.10.0'} - dev: false - - /decamelize/6.0.0: - resolution: {integrity: sha512-Fv96DCsdOgB6mdGl67MT5JaTNKRzrzill5OH5s8bjYJXVlcXyPYGyPsUkWyGV5p1TXI5esYIYMMeDJL0hEIwaA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dev: false - - /delayed-stream/1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - dev: false - - /error-ex/1.3.2: - resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} - dependencies: - is-arrayish: 0.2.1 - dev: false - - /escape-string-regexp/1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - dev: false - - /extract-files/9.0.0: - resolution: {integrity: sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ==} - engines: {node: ^10.17.0 || ^12.0.0 || >= 13.7.0} - dev: false - - /find-up/6.3.0: - resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - locate-path: 7.1.1 - path-exists: 5.0.0 - dev: false - - /form-data/3.0.1: - resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - - /function-bind/1.1.1: - resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - dev: false - - /graphql-request/4.3.0_graphql@16.6.0: - resolution: {integrity: sha512-2v6hQViJvSsifK606AliqiNiijb1uwWp6Re7o0RTyH+uRTv/u7Uqm2g4Fjq/LgZIzARB38RZEvVBFOQOVdlBow==} - peerDependencies: - graphql: 14 - 16 - dependencies: - cross-fetch: 3.1.5 - extract-files: 9.0.0 - form-data: 3.0.1 - graphql: 16.6.0 - transitivePeerDependencies: - - encoding - dev: false - - /graphql/16.6.0: - resolution: {integrity: sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==} - engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} - dev: false - - /hard-rejection/2.1.0: - resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} - engines: {node: '>=6'} - dev: false - - /has-flag/3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - dev: false - - /has/1.0.3: - resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} - engines: {node: '>= 0.4.0'} - dependencies: - function-bind: 1.1.1 - dev: false - - /hosted-git-info/4.1.0: - resolution: {integrity: sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==} - engines: {node: '>=10'} - dependencies: - lru-cache: 6.0.0 - dev: false - - /hosted-git-info/5.2.1: - resolution: {integrity: sha512-xIcQYMnhcx2Nr4JTjsFmwwnr9vldugPy9uVm0o87bjqqWMv9GaqsTeT+i99wTl0mk1uLxJtHxLb8kymqTENQsw==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - lru-cache: 7.14.1 - dev: false - - /indent-string/5.0.0: - resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} - engines: {node: '>=12'} - dev: false - - /is-arrayish/0.2.1: - resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} - dev: false - - /is-core-module/2.11.0: - resolution: {integrity: sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==} - dependencies: - has: 1.0.3 - dev: false - - /is-plain-obj/1.1.0: - resolution: {integrity: sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==} - engines: {node: '>=0.10.0'} - dev: false - - /js-tokens/4.0.0: - resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - dev: false - - /json-parse-even-better-errors/2.3.1: - resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} - dev: false - - /kind-of/6.0.3: - resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} - engines: {node: '>=0.10.0'} - dev: false - - /lines-and-columns/1.2.4: - resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - dev: false - - /locate-path/7.1.1: - resolution: {integrity: sha512-vJXaRMJgRVD3+cUZs3Mncj2mxpt5mP0EmNOsxRSZRMlbqjvxzDEOIUWXGmavo0ZC9+tNZCBLQ66reA11nbpHZg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - p-locate: 6.0.0 - dev: false - - /lru-cache/6.0.0: - resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} - engines: {node: '>=10'} - dependencies: - yallist: 4.0.0 - dev: false - - /lru-cache/7.14.1: - resolution: {integrity: sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==} - engines: {node: '>=12'} - dev: false - - /map-obj/1.0.1: - resolution: {integrity: sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==} - engines: {node: '>=0.10.0'} - dev: false - - /map-obj/4.3.0: - resolution: {integrity: sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==} - engines: {node: '>=8'} - dev: false - - /meow/11.0.0: - resolution: {integrity: sha512-Cl0yeeIrko6d94KpUo1M+0X1sB14ikoaqlIGuTH1fW4I+E3+YljL54/hb/BWmVfrV9tTV9zU04+xjw08Fh2WkA==} - engines: {node: '>=14.16'} - dependencies: - '@types/minimist': 1.2.2 - camelcase-keys: 8.0.2 - decamelize: 6.0.0 - decamelize-keys: 1.1.1 - hard-rejection: 2.1.0 - minimist-options: 4.1.0 - normalize-package-data: 4.0.1 - read-pkg-up: 9.1.0 - redent: 4.0.0 - trim-newlines: 4.0.2 - type-fest: 3.3.0 - yargs-parser: 21.1.1 - dev: false - - /mime-db/1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - dev: false - - /mime-types/2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - dependencies: - mime-db: 1.52.0 - dev: false - - /min-indent/1.0.1: - resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} - engines: {node: '>=4'} - dev: false - - /minimist-options/4.1.0: - resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} - engines: {node: '>= 6'} - dependencies: - arrify: 1.0.1 - is-plain-obj: 1.1.0 - kind-of: 6.0.3 - dev: false - - /node-fetch/2.6.7: - resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} - engines: {node: 4.x || >=6.0.0} - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - dependencies: - whatwg-url: 5.0.0 - dev: false - - /normalize-package-data/3.0.3: - resolution: {integrity: sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==} - engines: {node: '>=10'} - dependencies: - hosted-git-info: 4.1.0 - is-core-module: 2.11.0 - semver: 7.3.8 - validate-npm-package-license: 3.0.4 - dev: false - - /normalize-package-data/4.0.1: - resolution: {integrity: sha512-EBk5QKKuocMJhB3BILuKhmaPjI8vNRSpIfO9woLC6NyHVkKKdVEdAO1mrT0ZfxNR1lKwCcTkuZfmGIFdizZ8Pg==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - hosted-git-info: 5.2.1 - is-core-module: 2.11.0 - semver: 7.3.8 - validate-npm-package-license: 3.0.4 - dev: false - - /p-limit/4.0.0: - resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - yocto-queue: 1.0.0 - dev: false - - /p-locate/6.0.0: - resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - p-limit: 4.0.0 - dev: false - - /parse-json/5.2.0: - resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} - engines: {node: '>=8'} - dependencies: - '@babel/code-frame': 7.18.6 - error-ex: 1.3.2 - json-parse-even-better-errors: 2.3.1 - lines-and-columns: 1.2.4 - dev: false - - /path-exists/5.0.0: - resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dev: false - - /quick-lru/6.1.1: - resolution: {integrity: sha512-S27GBT+F0NTRiehtbrgaSE1idUAJ5bX8dPAQTdylEyNlrdcH5X4Lz7Edz3DYzecbsCluD5zO8ZNEe04z3D3u6Q==} - engines: {node: '>=12'} - dev: false - - /read-pkg-up/9.1.0: - resolution: {integrity: sha512-vaMRR1AC1nrd5CQM0PhlRsO5oc2AAigqr7cCrZ/MW/Rsaflz4RlgzkpL4qoU/z1F6wrbd85iFv1OQj/y5RdGvg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - find-up: 6.3.0 - read-pkg: 7.1.0 - type-fest: 2.19.0 - dev: false - - /read-pkg/7.1.0: - resolution: {integrity: sha512-5iOehe+WF75IccPc30bWTbpdDQLOCc3Uu8bi3Dte3Eueij81yx1Mrufk8qBx/YAbR4uL1FdUr+7BKXDwEtisXg==} - engines: {node: '>=12.20'} - dependencies: - '@types/normalize-package-data': 2.4.1 - normalize-package-data: 3.0.3 - parse-json: 5.2.0 - type-fest: 2.19.0 - dev: false - - /redent/4.0.0: - resolution: {integrity: sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==} - engines: {node: '>=12'} - dependencies: - indent-string: 5.0.0 - strip-indent: 4.0.0 - dev: false - - /semver/7.3.8: - resolution: {integrity: sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==} - engines: {node: '>=10'} - hasBin: true - dependencies: - lru-cache: 6.0.0 - dev: false - - /spdx-correct/3.1.1: - resolution: {integrity: sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==} - dependencies: - spdx-expression-parse: 3.0.1 - spdx-license-ids: 3.0.12 - dev: false - - /spdx-exceptions/2.3.0: - resolution: {integrity: sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==} - dev: false - - /spdx-expression-parse/3.0.1: - resolution: {integrity: sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==} - dependencies: - spdx-exceptions: 2.3.0 - spdx-license-ids: 3.0.12 - dev: false - - /spdx-license-ids/3.0.12: - resolution: {integrity: sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA==} - dev: false - - /strip-indent/4.0.0: - resolution: {integrity: sha512-mnVSV2l+Zv6BLpSD/8V87CW/y9EmmbYzGCIavsnsI6/nwn26DwffM/yztm30Z/I2DY9wdS3vXVCMnHDgZaVNoA==} - engines: {node: '>=12'} - dependencies: - min-indent: 1.0.1 - dev: false - - /supports-color/5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} - dependencies: - has-flag: 3.0.0 - dev: false - - /tr46/0.0.3: - resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - dev: false - - /trim-newlines/4.0.2: - resolution: {integrity: sha512-GJtWyq9InR/2HRiLZgpIKv+ufIKrVrvjQWEj7PxAXNc5dwbNJkqhAUoAGgzRmULAnoOM5EIpveYd3J2VeSAIew==} - engines: {node: '>=12'} - dev: false - - /type-fest/2.19.0: - resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} - engines: {node: '>=12.20'} - dev: false - - /type-fest/3.3.0: - resolution: {integrity: sha512-gezeeOIZyQLGW5uuCeEnXF1aXmtt2afKspXz3YqoOcZ3l/YMJq1pujvgT+cz/Nw1O/7q/kSav5fihJHsC/AOUg==} - engines: {node: '>=14.16'} - dev: false - - /validate-npm-package-license/3.0.4: - resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} - dependencies: - spdx-correct: 3.1.1 - spdx-expression-parse: 3.0.1 - dev: false - - /webidl-conversions/3.0.1: - resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - dev: false - - /whatwg-url/5.0.0: - resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} - dependencies: - tr46: 0.0.3 - webidl-conversions: 3.0.1 - dev: false - - /yallist/4.0.0: - resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: false - - /yargs-parser/21.1.1: - resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} - engines: {node: '>=12'} - dev: false - - /yocto-queue/1.0.0: - resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} - engines: {node: '>=12.20'} - dev: false diff --git a/scripts/seed-database/seed-db.sh b/scripts/seed-database/seed-db.sh deleted file mode 100644 index 7294bace8a..0000000000 --- a/scripts/seed-database/seed-db.sh +++ /dev/null @@ -1,7 +0,0 @@ -cd /hasura/ - -hasura seeds apply - -cd /scripts/ - -pnpm upsert-flows diff --git a/scripts/seed-database/sync.sh b/scripts/seed-database/sync.sh new file mode 100755 index 0000000000..5cd1011c2c --- /dev/null +++ b/scripts/seed-database/sync.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +set -xe + +SCRIPT_DIR=$(dirname "$0") + +# Check whether `docker compose` or `docker-compose` is available +if command -v docker-compose > /dev/null; then + DOCKER_COMPOSE=docker-compose +elif command -v docker compose > /dev/null; then + DOCKER_COMPOSE="docker compose" +else + echo "Neither docker compose nor docker-compose is available" + exit 1 +fi + +REMOTE_PG="${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" +LOCAL_PG="postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres/${PG_DATABASE}" + +cd "${SCRIPT_DIR}/../../" || exit +${DOCKER_COMPOSE} run --rm -v "$(pwd):/app" postgres /app/scripts/seed-database/container.sh "${LOCAL_PG}" "${REMOTE_PG}" +cd - || exit 1 diff --git a/scripts/seed-database/upsert-production-flows.js b/scripts/seed-database/upsert-production-flows.js deleted file mode 100644 index 4826b538d9..0000000000 --- a/scripts/seed-database/upsert-production-flows.js +++ /dev/null @@ -1,259 +0,0 @@ -import { gql, GraphQLClient } from "graphql-request"; -import pLimit from 'p-limit'; -import meow from 'meow'; - -const DEFAULT_LIMIT = 10; - -const cli = meow(` - Options - --overwrite, -o Overwrites existing flows and teams - --limit, -l Rate limit for published flows insertion (Default: ${DEFAULT_LIMIT}) -`, { - importMeta: import.meta, - flags: { - overwrite: { - type: 'boolean', - alias: 'o', - }, - limit: { - type: 'number', - alias: 'l' - } - } -}); - -const PRODUCTION_GRAPHQL_URL = 'https://hasura.editor.planx.uk/v1/graphql'; -const LOCAL_GRAPHQL_URL = process.env.HASURA_GRAPHQL_URL; -const LOCAL_GRAPHQL_ADMIN_SECRET = process.env.HASURA_GRAPHQL_ADMIN_SECRET; - -(async () => { - try { - const productionClient = new GraphQLClient(PRODUCTION_GRAPHQL_URL); - const localClient = new GraphQLClient(LOCAL_GRAPHQL_URL, { - headers: { - "x-hasura-admin-secret": LOCAL_GRAPHQL_ADMIN_SECRET, - }, - }); - - const shouldOverwrite = cli.flags.overwrite; - const rateLimit = cli.flags.limit || DEFAULT_LIMIT; - - if (!shouldOverwrite) { - const { flows: localFlows } = await localClient.request(gql` - query GetAllFlows { - flows { - id - } - } - `); - - if (localFlows.length > 0) { - // If a flow belongs to an existing team we can not delete the teams because of existing constraints. - console.log('There are flows in the local database, refusing to continue.'); - process.exit(0) - } - } - - console.log('Fetching teams and flows...'); - const { - flows: productionFlows, - teams: productionTeams, - } = await productionClient.request(gql` - query GetAllFlowsAndTeams { - flows { - id - data - slug - team_id - settings - created_at - updated_at - } - teams { - id - name - slug - theme - settings - notify_personalisation - } - } - `); - - await localClient.request(deleteFlowsAndTeamsQuery); - - console.log("Inserting flows and teams..."); - - await localClient.request( - insertTeamsMutation, - { teams: productionTeams || [] } - ); - - await localClient.request( - insertFlowsMutation, - { - flows: productionFlows?.map(flow => ({ - ...flow, - version: 1, - })) || [], - } - ); - - console.log('Inserting Operations...'); - - await insertOperations(localClient)(productionFlows || []); - - console.log('Fetching published flows...'); - // throttling is needed to prevent errors when fetching a large amount of data - const limit = pLimit(rateLimit); - const rateLimitedSync = (flowId) => - limit(() => syncPublishedFlow(productionClient, localClient, flowId)); - - await Promise.all(productionFlows.map(async flow => rateLimitedSync(flow.id))); - - console.log("Production flows and teams inserted successfully."); - } catch (err) { - console.log(err) - console.error('It was not possible to insert flows and teams.') - process.exit(1) - } -})() - -const deleteFlowsAndTeamsQuery = gql` - mutation CleanDatabase { - delete_analytics_logs(where: {}){ - affected_rows - } - delete_analytics(where: {}){ - affected_rows - } - delete_published_flows(where: {}){ - affected_rows - } - delete_operations(where: {}){ - affected_rows - } - delete_flows(where: {}) { - affected_rows - } - delete_teams(where: {}) { - affected_rows - } - } -`; - -const insertTeamsMutation = gql` - mutation InsertTeams( - $teams: [teams_insert_input!]!, - ) { - insert_teams( - objects: $teams, - ) { - affected_rows - } - } -`; - -const insertFlowsMutation = gql` - mutation InsertFlows( - $flows: [flows_insert_input!]!, - ) { - insert_flows( - objects: $flows, - ) { - affected_rows - } - } -`; - -const syncPublishedFlow = async (productionClient, localClient, flowId) => { - const publishedFlows = await publishedFlowsByFlowIdQuery(flowId, productionClient); - - console.log(`Inserting ${publishedFlows.length} published flows for flow id ${flowId}`); - - await Promise.all(publishedFlows.map( - async publishedFlow => insertPublishedFlowMutation( - { - ...publishedFlow, - publisher_id: 1 // prevents errors with non-existing ID constraint - }, - localClient - ) - )); -} - -const publishedFlowsByFlowIdQuery = async (flowId, graphQLClient) => { - try { - const data = await graphQLClient.request(gql` - query GetPublishedFlowByFlowId($id: uuid!) { - published_flows( - where: {flow_id: {_eq: $id}}, - limit: 2, - order_by: {created_at: desc} - ) { - id - data - flow_id - summary - } - }`, - { id: flowId } - ); - - return data.published_flows; - } catch (err) { - console.log(`Error fetching publishedFlow id ${flowId}`); - throw err; - } -} - -const insertPublishedFlowMutation = (publishedFlow, graphQLClient) => { - return graphQLClient.request(gql` - mutation InsertPublishedFlow( - $publishedFlow: published_flows_insert_input! - ) { - insert_published_flows_one( - object: $publishedFlow - ) { - id - } - } - `, { - publishedFlow, - }).catch((err) => { - console.log(`Error inserting published flow ${publishedFlow?.id}`); - throw err - }); -} - -const insertOperations = (localClient) => async (flows) => { - const operations = flows.map(flow => buildOperationPayload(flow)); - - await localClient.request(gql` - mutation InsertOperations($operations: [operations_insert_input!]!) { - insert_operations(objects: $operations) { - affected_rows - } - } - `, { - operations, - }) -} - -const buildOperationPayload = (flow) => ({ - flow_id: flow.id, - data: { - m: { - ts: Date.now(), - uId: "1" - }, - v: 0, - seq: 1, - src: "1", - create: { - data: {}, - type: "http://sharejs.org/types/JSONv0" - } - }, - version: 1, -}) diff --git a/scripts/upsert-flows.sh b/scripts/upsert-flows.sh deleted file mode 100755 index ed754adfdb..0000000000 --- a/scripts/upsert-flows.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -xe - -# Setup -pipe=/tmp/osl-partial-dump -rm -f $pipe -trap 'rm -f $pipe' EXIT -mkfifo $pipe - -# Download from DigitalOcean -ssh root@docker.planx.uk 'bash -s' <