diff --git a/.github/workflows/sync-staging-db.yml b/.github/workflows/sync-staging-db.yml index 4baa0ac016..0fe2ea5222 100644 --- a/.github/workflows/sync-staging-db.yml +++ b/.github/workflows/sync-staging-db.yml @@ -15,18 +15,24 @@ jobs: - name: Install pulumi uses: pulumi/setup-pulumi@v2 - run: | - echo "HASURA_GRAPHQL_ADMIN_SECRET=$(pulumi config get hasura-admin-secret --stack staging)" >> $GITHUB_ENV - working-directory: infrastructure/application + echo "STAGING_PG_URL=$(pulumi stack output --stack staging --show-secrets dbRootUrl)" >> $GITHUB_ENV + working-directory: infrastructure/data env: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} - - uses: actions/setup-node@v2 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 with: - node-version: ${{ env.NODE_VERSION }} - - run: npm install -g pnpm - working-directory: scripts/seed-database - - run: pnpm install - working-directory: scripts/seed-database - - run: node ./upsert-production-flows.js --overwrite --limit 1 - working-directory: scripts/seed-database + aws-access-key-id: ${{ secrets.PIZZA_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.PIZZA_AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + - name: Copy .env files from Staging S3 to the current working directory with AWS CLI + run: ./scripts/pull-secrets.sh + - name: Check if .env files exist + id: check_files + uses: andstor/file-existence-action@v1 + with: + files: ".env, .env.staging, api.planx.uk/.env.test, hasura.planx.uk/.env.test" + fail: true + - run: docker run --rm -v "./scripts/seed-database:/app" --workdir="/app" postgis/postgis:12-3.0-alpine "./container.sh ${STAGING_PG_URL} ${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" env: - HASURA_GRAPHQL_URL: https://hasura.editor.planx.dev/v1/graphql + PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS: ${{ secrets.PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS }} diff --git a/docker-compose.yml b/docker-compose.yml index d6e64124bc..2ac9ccaab7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,20 +96,16 @@ services: retries: 3 seed-database: - build: - context: ./scripts/seed-database + image: postgis/postgis:12-3.0-alpine volumes: - - "./hasura.planx.uk/:/hasura" + - "./scripts/seed-database:/app" + working_dir: "/app" + entrypoint: sh + command: -c "./container.sh postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres/${PG_DATABASE} ${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" restart: "no" depends_on: hasura-proxy: condition: service_healthy - environment: - HASURA_GRAPHQL_ADMIN_SECRET: ${HASURA_GRAPHQL_ADMIN_SECRET} - HASURA_GRAPHQL_URL: http://hasura-proxy:7000/v1/graphql - HASURA_GRAPHQL_ENDPOINT: http://hasura-proxy:7000/ - PRODUCTION_GRAPHQL_URL: 'https://hasura.editor.planx.uk/v1/graphql' - api: restart: unless-stopped build: diff --git a/scripts/pullrequest/create.sh b/scripts/pullrequest/create.sh index 6fb425eb87..5e61434b84 100755 --- a/scripts/pullrequest/create.sh +++ b/scripts/pullrequest/create.sh @@ -21,9 +21,6 @@ echo \ apt-get update -y apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y -# install hasura cli -curl -L https://github.com/hasura/graphql-engine/raw/stable/cli/get.sh | bash - # set env for this shell set -o allexport source .env.pizza @@ -35,7 +32,3 @@ docker compose \ -f docker-compose.yml \ -f docker-compose.pizza.yml \ up --build --wait - -# insert hasura seeds -cd hasura.planx.uk -hasura seed apply --envfile .env diff --git a/scripts/seed-database/.dockerignore b/scripts/seed-database/.dockerignore deleted file mode 100644 index c2658d7d1b..0000000000 --- a/scripts/seed-database/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -node_modules/ diff --git a/scripts/seed-database/Dockerfile b/scripts/seed-database/Dockerfile deleted file mode 100644 index de52a5b616..0000000000 --- a/scripts/seed-database/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM node:16.13.1-alpine -RUN npm install -g hasura-cli - -WORKDIR /scripts/ -RUN npm install -g pnpm@7.8.0 -COPY pnpm-lock.yaml . - -RUN pnpm fetch --prod -ADD . . -RUN pnpm install -f --recursive --offline --prod -COPY . . -CMD ["sh", "./seed-db.sh"] \ No newline at end of file diff --git a/scripts/seed-database/README.md b/scripts/seed-database/README.md new file mode 100644 index 0000000000..0ea45a0065 --- /dev/null +++ b/scripts/seed-database/README.md @@ -0,0 +1,5 @@ +# seed-database + +This script uses a read-only database user to selectively sync data from a production database to a local development database. + +This is useful for having production-grade data both locally and on ephemeral pizza links (i.e. vultr servers that are spun up for each Pull Request). diff --git a/scripts/seed-database/container.sh b/scripts/seed-database/container.sh new file mode 100755 index 0000000000..224635fbd6 --- /dev/null +++ b/scripts/seed-database/container.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# This is the script that runs inside the container +# Usage: container.sh + +# cd to this script's directory +cd "$(dirname "$0")" || exit + +set -ex + +LOCAL_PG="$1" +REMOTE_PG="$2" + +# fetch users +psql --command="\\COPY (SELECT * FROM users) TO '/tmp/users.csv' (FORMAT CSV, DELIMITER ';')" "${REMOTE_PG}" + +# fetch teams +psql --command="\\COPY (SELECT id, name, slug, theme, settings, domain FROM teams) TO '/tmp/teams.csv' (FORMAT CSV, DELIMITER ';')" "${REMOTE_PG}" + +# fetch flows +psql --command="\\COPY (SELECT * FROM flows) TO '/tmp/flows.csv' (FORMAT CSV, DELIMITER ';')" "${REMOTE_PG}" + +# fetch published_flows (the last two) +psql --command="\\COPY (SELECT id, data, flow_id, summary, publisher_id FROM (SELECT id, data, flow_id, summary, publisher_id, ROW_NUMBER() OVER (PARTITION BY flow_id ORDER BY created_at DESC) as row_num FROM published_flows) as subquery WHERE row_num <= 2) TO '/tmp/published_flows.csv' (FORMAT CSV);" "${REMOTE_PG}" + +# run container.sql +psql "${LOCAL_PG}" < container.sql diff --git a/scripts/seed-database/container.sql b/scripts/seed-database/container.sql new file mode 100644 index 0000000000..57f42b8b52 --- /dev/null +++ b/scripts/seed-database/container.sql @@ -0,0 +1,34 @@ +BEGIN; + +TRUNCATE TABLE users, teams CASCADE; + +\COPY users FROM '/tmp/users.csv' (FORMAT CSV, DELIMITER ';'); + +\COPY teams (id, name, slug, theme, settings, domain) FROM '/tmp/teams.csv' (FORMAT CSV, DELIMITER ';') + +\COPY flows FROM '/tmp/flows.csv' (FORMAT CSV, DELIMITER ';'); +UPDATE flows SET version = 1; + +-- insert an operation for each flow (to make sharedb happy) +INSERT INTO operations (flow_id, data, version) + SELECT + id + , json_build_object( + 'm', json_build_object( + 'ts', extract(epoch from now()) * 1000 + , 'uId', '1' + ) + , 'v', 0 + , 'seq', 1 + , 'src', '1' + , 'create', json_build_object( + 'data', '{}' + , 'type', 'http://sharejs.org/types/JSONv0' + ) + ) + , 1 + FROM flows; + +\COPY published_flows (id, data, flow_id, summary, publisher_id) FROM '/tmp/published_flows.csv' (FORMAT CSV); + +COMMIT; diff --git a/scripts/seed-database/package.json b/scripts/seed-database/package.json deleted file mode 100644 index 1e9d394f61..0000000000 --- a/scripts/seed-database/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "seed-database", - "private": true, - "license": "MPL-2.0", - "type": "module", - "scripts": { - "upsert-flows": "node --experimental-specifier-resolution=node ./upsert-production-flows" - }, - "engines": { - "node": "^16", - "pnpm": "^7.8.0" - }, - "dependencies": { - "graphql": "^16.5.0", - "graphql-request": "^4.3.0", - "meow": "^11.0.0", - "p-limit": "^4.0.0" - }, - "devDependencies": { - "@types/lodash": "^4.14.184" - } -} \ No newline at end of file diff --git a/scripts/seed-database/pnpm-lock.yaml b/scripts/seed-database/pnpm-lock.yaml deleted file mode 100644 index 4664f85839..0000000000 --- a/scripts/seed-database/pnpm-lock.yaml +++ /dev/null @@ -1,524 +0,0 @@ -lockfileVersion: 5.4 - -specifiers: - '@types/lodash': ^4.14.184 - graphql: ^16.5.0 - graphql-request: ^4.3.0 - meow: ^11.0.0 - p-limit: ^4.0.0 - -dependencies: - graphql: 16.6.0 - graphql-request: 4.3.0_graphql@16.6.0 - meow: 11.0.0 - p-limit: 4.0.0 - -devDependencies: - '@types/lodash': 4.14.184 - -packages: - - /@babel/code-frame/7.18.6: - resolution: {integrity: sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/highlight': 7.18.6 - dev: false - - /@babel/helper-validator-identifier/7.19.1: - resolution: {integrity: sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==} - engines: {node: '>=6.9.0'} - dev: false - - /@babel/highlight/7.18.6: - resolution: {integrity: sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==} - engines: {node: '>=6.9.0'} - dependencies: - '@babel/helper-validator-identifier': 7.19.1 - chalk: 2.4.2 - js-tokens: 4.0.0 - dev: false - - /@types/lodash/4.14.184: - resolution: {integrity: sha512-RoZphVtHbxPZizt4IcILciSWiC6dcn+eZ8oX9IWEYfDMcocdd42f7NPI6fQj+6zI8y4E0L7gu2pcZKLGTRaV9Q==} - dev: true - - /@types/minimist/1.2.2: - resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} - dev: false - - /@types/normalize-package-data/2.4.1: - resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} - dev: false - - /ansi-styles/3.2.1: - resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} - engines: {node: '>=4'} - dependencies: - color-convert: 1.9.3 - dev: false - - /arrify/1.0.1: - resolution: {integrity: sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==} - engines: {node: '>=0.10.0'} - dev: false - - /asynckit/0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: false - - /camelcase-keys/8.0.2: - resolution: {integrity: sha512-qMKdlOfsjlezMqxkUGGMaWWs17i2HoL15tM+wtx8ld4nLrUwU58TFdvyGOz/piNP842KeO8yXvggVQSdQ828NA==} - engines: {node: '>=14.16'} - dependencies: - camelcase: 7.0.0 - map-obj: 4.3.0 - quick-lru: 6.1.1 - type-fest: 2.19.0 - dev: false - - /camelcase/7.0.0: - resolution: {integrity: sha512-JToIvOmz6nhGsUhAYScbo2d6Py5wojjNfoxoc2mEVLUdJ70gJK2gnd+ABY1Tc3sVMyK7QDPtN0T/XdlCQWITyQ==} - engines: {node: '>=14.16'} - dev: false - - /chalk/2.4.2: - resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} - engines: {node: '>=4'} - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - dev: false - - /color-convert/1.9.3: - resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} - dependencies: - color-name: 1.1.3 - dev: false - - /color-name/1.1.3: - resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - dev: false - - /combined-stream/1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - dependencies: - delayed-stream: 1.0.0 - dev: false - - /cross-fetch/3.1.5: - resolution: {integrity: sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==} - dependencies: - node-fetch: 2.6.7 - transitivePeerDependencies: - - encoding - dev: false - - /decamelize-keys/1.1.1: - resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} - engines: {node: '>=0.10.0'} - dependencies: - decamelize: 1.2.0 - map-obj: 1.0.1 - dev: false - - /decamelize/1.2.0: - resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} - engines: {node: '>=0.10.0'} - dev: false - - /decamelize/6.0.0: - resolution: {integrity: sha512-Fv96DCsdOgB6mdGl67MT5JaTNKRzrzill5OH5s8bjYJXVlcXyPYGyPsUkWyGV5p1TXI5esYIYMMeDJL0hEIwaA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dev: false - - /delayed-stream/1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - dev: false - - /error-ex/1.3.2: - resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} - dependencies: - is-arrayish: 0.2.1 - dev: false - - /escape-string-regexp/1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - dev: false - - /extract-files/9.0.0: - resolution: {integrity: sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ==} - engines: {node: ^10.17.0 || ^12.0.0 || >= 13.7.0} - dev: false - - /find-up/6.3.0: - resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - locate-path: 7.1.1 - path-exists: 5.0.0 - dev: false - - /form-data/3.0.1: - resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - - /function-bind/1.1.1: - resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - dev: false - - /graphql-request/4.3.0_graphql@16.6.0: - resolution: {integrity: sha512-2v6hQViJvSsifK606AliqiNiijb1uwWp6Re7o0RTyH+uRTv/u7Uqm2g4Fjq/LgZIzARB38RZEvVBFOQOVdlBow==} - peerDependencies: - graphql: 14 - 16 - dependencies: - cross-fetch: 3.1.5 - extract-files: 9.0.0 - form-data: 3.0.1 - graphql: 16.6.0 - transitivePeerDependencies: - - encoding - dev: false - - /graphql/16.6.0: - resolution: {integrity: sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==} - engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} - dev: false - - /hard-rejection/2.1.0: - resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} - engines: {node: '>=6'} - dev: false - - /has-flag/3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - dev: false - - /has/1.0.3: - resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} - engines: {node: '>= 0.4.0'} - dependencies: - function-bind: 1.1.1 - dev: false - - /hosted-git-info/4.1.0: - resolution: {integrity: sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==} - engines: {node: '>=10'} - dependencies: - lru-cache: 6.0.0 - dev: false - - /hosted-git-info/5.2.1: - resolution: {integrity: sha512-xIcQYMnhcx2Nr4JTjsFmwwnr9vldugPy9uVm0o87bjqqWMv9GaqsTeT+i99wTl0mk1uLxJtHxLb8kymqTENQsw==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - lru-cache: 7.14.1 - dev: false - - /indent-string/5.0.0: - resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} - engines: {node: '>=12'} - dev: false - - /is-arrayish/0.2.1: - resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} - dev: false - - /is-core-module/2.11.0: - resolution: {integrity: sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==} - dependencies: - has: 1.0.3 - dev: false - - /is-plain-obj/1.1.0: - resolution: {integrity: sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==} - engines: {node: '>=0.10.0'} - dev: false - - /js-tokens/4.0.0: - resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} - dev: false - - /json-parse-even-better-errors/2.3.1: - resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} - dev: false - - /kind-of/6.0.3: - resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} - engines: {node: '>=0.10.0'} - dev: false - - /lines-and-columns/1.2.4: - resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} - dev: false - - /locate-path/7.1.1: - resolution: {integrity: sha512-vJXaRMJgRVD3+cUZs3Mncj2mxpt5mP0EmNOsxRSZRMlbqjvxzDEOIUWXGmavo0ZC9+tNZCBLQ66reA11nbpHZg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - p-locate: 6.0.0 - dev: false - - /lru-cache/6.0.0: - resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} - engines: {node: '>=10'} - dependencies: - yallist: 4.0.0 - dev: false - - /lru-cache/7.14.1: - resolution: {integrity: sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==} - engines: {node: '>=12'} - dev: false - - /map-obj/1.0.1: - resolution: {integrity: sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==} - engines: {node: '>=0.10.0'} - dev: false - - /map-obj/4.3.0: - resolution: {integrity: sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==} - engines: {node: '>=8'} - dev: false - - /meow/11.0.0: - resolution: {integrity: sha512-Cl0yeeIrko6d94KpUo1M+0X1sB14ikoaqlIGuTH1fW4I+E3+YljL54/hb/BWmVfrV9tTV9zU04+xjw08Fh2WkA==} - engines: {node: '>=14.16'} - dependencies: - '@types/minimist': 1.2.2 - camelcase-keys: 8.0.2 - decamelize: 6.0.0 - decamelize-keys: 1.1.1 - hard-rejection: 2.1.0 - minimist-options: 4.1.0 - normalize-package-data: 4.0.1 - read-pkg-up: 9.1.0 - redent: 4.0.0 - trim-newlines: 4.0.2 - type-fest: 3.3.0 - yargs-parser: 21.1.1 - dev: false - - /mime-db/1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - dev: false - - /mime-types/2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - dependencies: - mime-db: 1.52.0 - dev: false - - /min-indent/1.0.1: - resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} - engines: {node: '>=4'} - dev: false - - /minimist-options/4.1.0: - resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} - engines: {node: '>= 6'} - dependencies: - arrify: 1.0.1 - is-plain-obj: 1.1.0 - kind-of: 6.0.3 - dev: false - - /node-fetch/2.6.7: - resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} - engines: {node: 4.x || >=6.0.0} - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - dependencies: - whatwg-url: 5.0.0 - dev: false - - /normalize-package-data/3.0.3: - resolution: {integrity: sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==} - engines: {node: '>=10'} - dependencies: - hosted-git-info: 4.1.0 - is-core-module: 2.11.0 - semver: 7.3.8 - validate-npm-package-license: 3.0.4 - dev: false - - /normalize-package-data/4.0.1: - resolution: {integrity: sha512-EBk5QKKuocMJhB3BILuKhmaPjI8vNRSpIfO9woLC6NyHVkKKdVEdAO1mrT0ZfxNR1lKwCcTkuZfmGIFdizZ8Pg==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - hosted-git-info: 5.2.1 - is-core-module: 2.11.0 - semver: 7.3.8 - validate-npm-package-license: 3.0.4 - dev: false - - /p-limit/4.0.0: - resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - yocto-queue: 1.0.0 - dev: false - - /p-locate/6.0.0: - resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - p-limit: 4.0.0 - dev: false - - /parse-json/5.2.0: - resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} - engines: {node: '>=8'} - dependencies: - '@babel/code-frame': 7.18.6 - error-ex: 1.3.2 - json-parse-even-better-errors: 2.3.1 - lines-and-columns: 1.2.4 - dev: false - - /path-exists/5.0.0: - resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dev: false - - /quick-lru/6.1.1: - resolution: {integrity: sha512-S27GBT+F0NTRiehtbrgaSE1idUAJ5bX8dPAQTdylEyNlrdcH5X4Lz7Edz3DYzecbsCluD5zO8ZNEe04z3D3u6Q==} - engines: {node: '>=12'} - dev: false - - /read-pkg-up/9.1.0: - resolution: {integrity: sha512-vaMRR1AC1nrd5CQM0PhlRsO5oc2AAigqr7cCrZ/MW/Rsaflz4RlgzkpL4qoU/z1F6wrbd85iFv1OQj/y5RdGvg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - find-up: 6.3.0 - read-pkg: 7.1.0 - type-fest: 2.19.0 - dev: false - - /read-pkg/7.1.0: - resolution: {integrity: sha512-5iOehe+WF75IccPc30bWTbpdDQLOCc3Uu8bi3Dte3Eueij81yx1Mrufk8qBx/YAbR4uL1FdUr+7BKXDwEtisXg==} - engines: {node: '>=12.20'} - dependencies: - '@types/normalize-package-data': 2.4.1 - normalize-package-data: 3.0.3 - parse-json: 5.2.0 - type-fest: 2.19.0 - dev: false - - /redent/4.0.0: - resolution: {integrity: sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==} - engines: {node: '>=12'} - dependencies: - indent-string: 5.0.0 - strip-indent: 4.0.0 - dev: false - - /semver/7.3.8: - resolution: {integrity: sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==} - engines: {node: '>=10'} - hasBin: true - dependencies: - lru-cache: 6.0.0 - dev: false - - /spdx-correct/3.1.1: - resolution: {integrity: sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==} - dependencies: - spdx-expression-parse: 3.0.1 - spdx-license-ids: 3.0.12 - dev: false - - /spdx-exceptions/2.3.0: - resolution: {integrity: sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==} - dev: false - - /spdx-expression-parse/3.0.1: - resolution: {integrity: sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==} - dependencies: - spdx-exceptions: 2.3.0 - spdx-license-ids: 3.0.12 - dev: false - - /spdx-license-ids/3.0.12: - resolution: {integrity: sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA==} - dev: false - - /strip-indent/4.0.0: - resolution: {integrity: sha512-mnVSV2l+Zv6BLpSD/8V87CW/y9EmmbYzGCIavsnsI6/nwn26DwffM/yztm30Z/I2DY9wdS3vXVCMnHDgZaVNoA==} - engines: {node: '>=12'} - dependencies: - min-indent: 1.0.1 - dev: false - - /supports-color/5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} - dependencies: - has-flag: 3.0.0 - dev: false - - /tr46/0.0.3: - resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - dev: false - - /trim-newlines/4.0.2: - resolution: {integrity: sha512-GJtWyq9InR/2HRiLZgpIKv+ufIKrVrvjQWEj7PxAXNc5dwbNJkqhAUoAGgzRmULAnoOM5EIpveYd3J2VeSAIew==} - engines: {node: '>=12'} - dev: false - - /type-fest/2.19.0: - resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} - engines: {node: '>=12.20'} - dev: false - - /type-fest/3.3.0: - resolution: {integrity: sha512-gezeeOIZyQLGW5uuCeEnXF1aXmtt2afKspXz3YqoOcZ3l/YMJq1pujvgT+cz/Nw1O/7q/kSav5fihJHsC/AOUg==} - engines: {node: '>=14.16'} - dev: false - - /validate-npm-package-license/3.0.4: - resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} - dependencies: - spdx-correct: 3.1.1 - spdx-expression-parse: 3.0.1 - dev: false - - /webidl-conversions/3.0.1: - resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - dev: false - - /whatwg-url/5.0.0: - resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} - dependencies: - tr46: 0.0.3 - webidl-conversions: 3.0.1 - dev: false - - /yallist/4.0.0: - resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - dev: false - - /yargs-parser/21.1.1: - resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} - engines: {node: '>=12'} - dev: false - - /yocto-queue/1.0.0: - resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} - engines: {node: '>=12.20'} - dev: false diff --git a/scripts/seed-database/seed-db.sh b/scripts/seed-database/seed-db.sh deleted file mode 100644 index 7294bace8a..0000000000 --- a/scripts/seed-database/seed-db.sh +++ /dev/null @@ -1,7 +0,0 @@ -cd /hasura/ - -hasura seeds apply - -cd /scripts/ - -pnpm upsert-flows diff --git a/scripts/seed-database/sync.sh b/scripts/seed-database/sync.sh new file mode 100755 index 0000000000..5cd1011c2c --- /dev/null +++ b/scripts/seed-database/sync.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +set -xe + +SCRIPT_DIR=$(dirname "$0") + +# Check whether `docker compose` or `docker-compose` is available +if command -v docker-compose > /dev/null; then + DOCKER_COMPOSE=docker-compose +elif command -v docker compose > /dev/null; then + DOCKER_COMPOSE="docker compose" +else + echo "Neither docker compose nor docker-compose is available" + exit 1 +fi + +REMOTE_PG="${PRODUCTION_PG_URL_FOR_USER_GITHUB_ACTIONS}" +LOCAL_PG="postgres://${PG_USERNAME}:${PG_PASSWORD}@postgres/${PG_DATABASE}" + +cd "${SCRIPT_DIR}/../../" || exit +${DOCKER_COMPOSE} run --rm -v "$(pwd):/app" postgres /app/scripts/seed-database/container.sh "${LOCAL_PG}" "${REMOTE_PG}" +cd - || exit 1 diff --git a/scripts/seed-database/upsert-production-flows.js b/scripts/seed-database/upsert-production-flows.js deleted file mode 100644 index 4826b538d9..0000000000 --- a/scripts/seed-database/upsert-production-flows.js +++ /dev/null @@ -1,259 +0,0 @@ -import { gql, GraphQLClient } from "graphql-request"; -import pLimit from 'p-limit'; -import meow from 'meow'; - -const DEFAULT_LIMIT = 10; - -const cli = meow(` - Options - --overwrite, -o Overwrites existing flows and teams - --limit, -l Rate limit for published flows insertion (Default: ${DEFAULT_LIMIT}) -`, { - importMeta: import.meta, - flags: { - overwrite: { - type: 'boolean', - alias: 'o', - }, - limit: { - type: 'number', - alias: 'l' - } - } -}); - -const PRODUCTION_GRAPHQL_URL = 'https://hasura.editor.planx.uk/v1/graphql'; -const LOCAL_GRAPHQL_URL = process.env.HASURA_GRAPHQL_URL; -const LOCAL_GRAPHQL_ADMIN_SECRET = process.env.HASURA_GRAPHQL_ADMIN_SECRET; - -(async () => { - try { - const productionClient = new GraphQLClient(PRODUCTION_GRAPHQL_URL); - const localClient = new GraphQLClient(LOCAL_GRAPHQL_URL, { - headers: { - "x-hasura-admin-secret": LOCAL_GRAPHQL_ADMIN_SECRET, - }, - }); - - const shouldOverwrite = cli.flags.overwrite; - const rateLimit = cli.flags.limit || DEFAULT_LIMIT; - - if (!shouldOverwrite) { - const { flows: localFlows } = await localClient.request(gql` - query GetAllFlows { - flows { - id - } - } - `); - - if (localFlows.length > 0) { - // If a flow belongs to an existing team we can not delete the teams because of existing constraints. - console.log('There are flows in the local database, refusing to continue.'); - process.exit(0) - } - } - - console.log('Fetching teams and flows...'); - const { - flows: productionFlows, - teams: productionTeams, - } = await productionClient.request(gql` - query GetAllFlowsAndTeams { - flows { - id - data - slug - team_id - settings - created_at - updated_at - } - teams { - id - name - slug - theme - settings - notify_personalisation - } - } - `); - - await localClient.request(deleteFlowsAndTeamsQuery); - - console.log("Inserting flows and teams..."); - - await localClient.request( - insertTeamsMutation, - { teams: productionTeams || [] } - ); - - await localClient.request( - insertFlowsMutation, - { - flows: productionFlows?.map(flow => ({ - ...flow, - version: 1, - })) || [], - } - ); - - console.log('Inserting Operations...'); - - await insertOperations(localClient)(productionFlows || []); - - console.log('Fetching published flows...'); - // throttling is needed to prevent errors when fetching a large amount of data - const limit = pLimit(rateLimit); - const rateLimitedSync = (flowId) => - limit(() => syncPublishedFlow(productionClient, localClient, flowId)); - - await Promise.all(productionFlows.map(async flow => rateLimitedSync(flow.id))); - - console.log("Production flows and teams inserted successfully."); - } catch (err) { - console.log(err) - console.error('It was not possible to insert flows and teams.') - process.exit(1) - } -})() - -const deleteFlowsAndTeamsQuery = gql` - mutation CleanDatabase { - delete_analytics_logs(where: {}){ - affected_rows - } - delete_analytics(where: {}){ - affected_rows - } - delete_published_flows(where: {}){ - affected_rows - } - delete_operations(where: {}){ - affected_rows - } - delete_flows(where: {}) { - affected_rows - } - delete_teams(where: {}) { - affected_rows - } - } -`; - -const insertTeamsMutation = gql` - mutation InsertTeams( - $teams: [teams_insert_input!]!, - ) { - insert_teams( - objects: $teams, - ) { - affected_rows - } - } -`; - -const insertFlowsMutation = gql` - mutation InsertFlows( - $flows: [flows_insert_input!]!, - ) { - insert_flows( - objects: $flows, - ) { - affected_rows - } - } -`; - -const syncPublishedFlow = async (productionClient, localClient, flowId) => { - const publishedFlows = await publishedFlowsByFlowIdQuery(flowId, productionClient); - - console.log(`Inserting ${publishedFlows.length} published flows for flow id ${flowId}`); - - await Promise.all(publishedFlows.map( - async publishedFlow => insertPublishedFlowMutation( - { - ...publishedFlow, - publisher_id: 1 // prevents errors with non-existing ID constraint - }, - localClient - ) - )); -} - -const publishedFlowsByFlowIdQuery = async (flowId, graphQLClient) => { - try { - const data = await graphQLClient.request(gql` - query GetPublishedFlowByFlowId($id: uuid!) { - published_flows( - where: {flow_id: {_eq: $id}}, - limit: 2, - order_by: {created_at: desc} - ) { - id - data - flow_id - summary - } - }`, - { id: flowId } - ); - - return data.published_flows; - } catch (err) { - console.log(`Error fetching publishedFlow id ${flowId}`); - throw err; - } -} - -const insertPublishedFlowMutation = (publishedFlow, graphQLClient) => { - return graphQLClient.request(gql` - mutation InsertPublishedFlow( - $publishedFlow: published_flows_insert_input! - ) { - insert_published_flows_one( - object: $publishedFlow - ) { - id - } - } - `, { - publishedFlow, - }).catch((err) => { - console.log(`Error inserting published flow ${publishedFlow?.id}`); - throw err - }); -} - -const insertOperations = (localClient) => async (flows) => { - const operations = flows.map(flow => buildOperationPayload(flow)); - - await localClient.request(gql` - mutation InsertOperations($operations: [operations_insert_input!]!) { - insert_operations(objects: $operations) { - affected_rows - } - } - `, { - operations, - }) -} - -const buildOperationPayload = (flow) => ({ - flow_id: flow.id, - data: { - m: { - ts: Date.now(), - uId: "1" - }, - v: 0, - seq: 1, - src: "1", - create: { - data: {}, - type: "http://sharejs.org/types/JSONv0" - } - }, - version: 1, -}) diff --git a/scripts/upsert-flows.sh b/scripts/upsert-flows.sh deleted file mode 100755 index ed754adfdb..0000000000 --- a/scripts/upsert-flows.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -xe - -# Setup -pipe=/tmp/osl-partial-dump -rm -f $pipe -trap 'rm -f $pipe' EXIT -mkfifo $pipe - -# Download from DigitalOcean -ssh root@docker.planx.uk 'bash -s' <