From 22de1b89da2a2c8c30808882e313013da87deb02 Mon Sep 17 00:00:00 2001 From: reubensamuel <110553227+reubensamuel@users.noreply.github.com> Date: Tue, 14 Feb 2023 16:19:00 +0000 Subject: [PATCH] Revert "Revert "GAT-1887: Testing New Release from preprod to prod gateway-api"" --- .github/workflows/dev_deployment.yaml | 138 ++++++++++++++++++++++ .github/workflows/preprod_deployment.yaml | 55 +++++++++ .github/workflows/uat_deployment.yaml | 63 ++++++++++ Chart.yaml | 1 + src/config/server.js | 2 + src/resources/auth/auth.route.js | 2 +- src/resources/auth/utils.js | 2 +- src/resources/dataset/dataset.entity.js | 2 +- src/resources/metadata/metadata.route.js | 22 ++++ src/utils/datasetonboarding.util.js | 2 +- 10 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dev_deployment.yaml create mode 100644 .github/workflows/preprod_deployment.yaml create mode 100644 .github/workflows/uat_deployment.yaml create mode 100644 Chart.yaml create mode 100644 src/resources/metadata/metadata.route.js diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml new file mode 100644 index 000000000..3e3242a90 --- /dev/null +++ b/.github/workflows/dev_deployment.yaml @@ -0,0 +1,138 @@ +name: Build and Deploy to Cloud Run + +on: + push: + paths: + - 'Chart.yaml' + +env: + PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + +jobs: + build: + # needs: analyze + # Add 'id-token' with the intended permissions for workload identity federation + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: dev + + - name: Read VERSION file + id: getversion + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + # Used for production + # - uses: "marvinpinto/action-automatic-releases@latest" + # with: + # repo_token: "${{ secrets.GITHUB_TOKEN }}" + # automatic_release_tag: ${{ steps.getversion.outputs.version }} + # prerelease: false + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Login to GAR + uses: docker/login-action@v2 + with: + registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }} + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Build and Push Container + shell: bash + env: + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + + run: |- + docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./ + docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }} + # END - Docker auth and build + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben + deploy: + needs: build + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: dev + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Read VERSION file + id: getversion + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' + + with: + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + # If required, use the Cloud Run url output in later steps \ No newline at end of file diff --git a/.github/workflows/preprod_deployment.yaml b/.github/workflows/preprod_deployment.yaml new file mode 100644 index 000000000..b5e08ed1e --- /dev/null +++ b/.github/workflows/preprod_deployment.yaml @@ -0,0 +1,55 @@ +name: Deploy to Cloud Run + +on: + pull_request: + types: + - closed + branches: + - 'preprod' + +env: + PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + +jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben + deploy: + if: github.event.pull_request.merged == true + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-preprod + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: preprod + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Read VERSION file + id: getversion + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' + + with: + # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + # If required, use the Cloud Run url output in later steps \ No newline at end of file diff --git a/.github/workflows/uat_deployment.yaml b/.github/workflows/uat_deployment.yaml new file mode 100644 index 000000000..c4a1b1cbf --- /dev/null +++ b/.github/workflows/uat_deployment.yaml @@ -0,0 +1,63 @@ +name: Deploy to Cloud Run + +on: + pull_request: + types: + - closed + branches: + - 'release' + + +env: + PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id. + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location + +jobs: # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben. + # catchsha: + # uses: HDRUK/gateway-api/.github/workflows/dev_deployment.yaml@dev + deploy: + # if: github.event.pull_request.merged == true + permissions: + contents: write + id-token: write + + runs-on: ubuntu-latest + environment: legacy-uat + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: release + + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + + - name: Read VERSION file + id: getversion + # run: echo "::set-output name=version::$(cat Chart.yaml)" + run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT + + # - name: Get SHA + # id: getsha + # run: echo ${{ needs.catchsha.outputs.GITHUB_SHA }} + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME_API }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}' + + with: + # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:{{ steps.catchsha.outputs.GITHUB_SHA}} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + # Functionality not supported by Github Actions one to ccheck back agin in the future + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' \ No newline at end of file diff --git a/Chart.yaml b/Chart.yaml new file mode 100644 index 000000000..5eb312c66 --- /dev/null +++ b/Chart.yaml @@ -0,0 +1 @@ +v0.0.0 \ No newline at end of file diff --git a/src/config/server.js b/src/config/server.js index 60b5c46f2..57dcc6aa9 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -236,6 +236,8 @@ app.use('/api/v2/questionbank', require('../resources/questionbank/questionbank. app.use('/api/v2/data-use-registers', require('../resources/dataUseRegister/dataUseRegister.route')); app.use('/api/v1/locations', require('../resources/spatialfilter/SpatialRouter')); +app.use('/api/v1/metadata', require('../resources/metadata/metadata.route')); + initialiseAuthentication(app); // launch our backend into a port diff --git a/src/resources/auth/auth.route.js b/src/resources/auth/auth.route.js index fdc342f38..687959c47 100644 --- a/src/resources/auth/auth.route.js +++ b/src/resources/auth/auth.route.js @@ -25,7 +25,7 @@ router.get('/status', function (req, res, next) { if (err || !user) { return res.json({ success: true, - data: [{ role: 'Reader', id: null, name: null, loggedIn: false }], + data: [{ role: 'Reader', id: null, name: null, loggedIn: false, tempProp: true }], }); } else { // 1. Reformat teams array for frontend diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index 380d92432..231e6543b 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -129,7 +129,7 @@ const getTeams = async () => { const catchLoginErrorAndRedirect = (req, res, next) => { if (req.auth.err || !req.auth.user) { - if (req.auth.err === 'loginError') { + if (req.auth.err === 'loginError' || req.auth.user === undefined) { return res.status(200).redirect(process.env.homeURL + '/loginerror'); } diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index 2edc0815d..2e00900d0 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -50,7 +50,7 @@ export default class DatasetClass extends Entity { transformedObject.dataset['@schema'] = { type: `Dataset`, version: `2.0.0`, - url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json`, + url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/2.1.0/dataset.schema.json`, } // Return v2 object diff --git a/src/resources/metadata/metadata.route.js b/src/resources/metadata/metadata.route.js new file mode 100644 index 000000000..cc9e4596c --- /dev/null +++ b/src/resources/metadata/metadata.route.js @@ -0,0 +1,22 @@ +import express from 'express'; +import passport from 'passport'; + +import { utils } from '../auth'; +import { ROLES } from '../user/user.roles'; + +import datasetonboardingUtil from '../../utils/datasetonboarding.util'; + +const router = express.Router({ mergeParams: true }); + +router.post('/scoring', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin), async (req, res) => { + const { dataset } = req.body; + + if (!dataset) { + res.json({ success: false, error: 'Dataset object must be supplied and contain all required data', status: 400 }); + } + + const verdict = await datasetonboardingUtil.buildMetadataQuality(dataset, dataset.datasetv2, dataset.pid); + res.json({ success: true, data: verdict, status: 200 }); +}); + +module.exports = router; \ No newline at end of file diff --git a/src/utils/datasetonboarding.util.js b/src/utils/datasetonboarding.util.js index 139e79c7b..b151df000 100644 --- a/src/utils/datasetonboarding.util.js +++ b/src/utils/datasetonboarding.util.js @@ -758,7 +758,7 @@ const buildMetadataQuality = async (dataset, v2Object, pid) => { let rawdata = fs.readFileSync(__dirname + '/schema.json'); schema = JSON.parse(rawdata); - const ajv = new Ajv({ strict: false, allErrors: true }); + const ajv = new Ajv({ strict: false, allErrors: false }); addFormats(ajv); const validate = ajv.compile(schema); validate(cleanV2Object);