Merge pull request #939 from HDRUK/revert-938-revert-936-preprod

2 revert  "GAT-1887: Testing New Release from preprod to prod gateway-api""

.github/workflows/dev_deployment.yaml

@@ -0,0 +1,138 @@
+name: Build and Deploy to Cloud Run
+
+on:
+  push:
+    paths:
+    - 'Chart.yaml'
+
+env:
+  PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id.
+  GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location
+
+jobs:
+  build:
+    # needs: analyze
+    # Add 'id-token' with the intended permissions for workload identity federation
+    permissions:
+      contents: write
+      id-token: write
+
+    runs-on: ubuntu-latest
+    environment: legacy-dev
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: dev
+
+      - name: Read VERSION file
+        id: getversion
+        run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT
+      # Used for production
+      # - uses: "marvinpinto/action-automatic-releases@latest"
+      #   with:
+      #     repo_token: "${{ secrets.GITHUB_TOKEN }}"
+      #     automatic_release_tag: ${{ steps.getversion.outputs.version }}
+      #     prerelease: false    
+
+      - name: Google Auth
+        id: auth
+        uses: 'google-github-actions/auth@v0'
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' 
+          service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}'
+
+      - name: Login to GAR
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }}
+          username: oauth2accesstoken
+          password: ${{ steps.auth.outputs.access_token }}
+
+      - name: Build and Push Container
+        shell: bash
+        env: 
+          GAR_LOCATION: ${{ secrets.GAR_LOCATION }}
+          PROJECT_ID: ${{ secrets.PROJECT_ID }}
+          GAR_NAME: ${{ secrets.GAR_NAME_API }}
+
+        run: |-
+          docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./
+          docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}
+      # END - Docker auth and build
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+      # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben
+  deploy:
+    needs: build
+    permissions:
+      contents: write
+      id-token: write
+
+    runs-on: ubuntu-latest
+    environment: legacy-dev
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: dev
+
+      - name: Google Auth
+        id: auth
+        uses: 'google-github-actions/auth@v0'
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' 
+          service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}'
+
+      - name: Read VERSION file
+        id: getversion
+        run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT
+
+      - name: Deploy to Cloud Run
+        uses: actions-hub/gcloud@master
+        id: deploy
+        env:
+          PROJECT_ID: ${{ secrets.PROJECT_ID }}
+          GAR_LOCATION: ${{ secrets.GAR_LOCATION }}
+          GAR_NAME: ${{ secrets.GAR_NAME_API }}
+          SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}'
+          SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}'
+
+        with:
+          args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' 
+      # If required, use the Cloud Run url output in later steps

.github/workflows/preprod_deployment.yaml

@@ -0,0 +1,55 @@
+name: Deploy to Cloud Run
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - 'preprod'
+
+env:
+  PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id.
+  GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location
+
+jobs:      # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben
+  deploy:
+    if: github.event.pull_request.merged == true
+    permissions:
+      contents: write
+      id-token: write
+
+    runs-on: ubuntu-latest
+    environment: legacy-preprod
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: preprod
+
+      - name: Google Auth
+        id: auth
+        uses: 'google-github-actions/auth@v0'
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' 
+          service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}'
+
+      - name: Read VERSION file
+        id: getversion
+        run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT
+
+      - name: Deploy to Cloud Run
+        uses: actions-hub/gcloud@master
+        id: deploy
+        env:
+          PROJECT_ID: ${{ secrets.PROJECT_ID }}
+          DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }}
+          GAR_LOCATION: ${{ secrets.GAR_LOCATION }}
+          GAR_NAME: ${{ secrets.GAR_NAME_API }}
+          SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}'
+          SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}'
+
+        with:
+          # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' 
+          args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}'
+      # If required, use the Cloud Run url output in later steps

.github/workflows/uat_deployment.yaml

@@ -0,0 +1,63 @@
+name: Deploy to Cloud Run
+
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - 'release'
+
+
+env:
+  PROJECT_ID: '${{ secrets.PROJECT_ID }}' # TODO: update Google Cloud project id.
+  GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' # TODO: update Artifact Registry location
+
+jobs:      # Deployment please don't modify anything here as the infrastructure is controlled by terraform any changes here please agree with chris and reuben.
+  # catchsha:
+  #   uses: HDRUK/gateway-api/.github/workflows/dev_deployment.yaml@dev
+  deploy:
+    # if: github.event.pull_request.merged == true
+    permissions:
+      contents: write
+      id-token: write
+
+    runs-on: ubuntu-latest
+    environment: legacy-uat
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: release
+
+      - name: Google Auth
+        id: auth
+        uses: 'google-github-actions/auth@v0'
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' 
+          service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}'
+
+      - name: Read VERSION file
+        id: getversion
+        # run: echo "::set-output name=version::$(cat Chart.yaml)"
+        run: echo "version=$(cat Chart.yaml)" >> $GITHUB_OUTPUT
+
+      # - name: Get SHA
+      #   id: getsha
+      #   run: echo ${{ needs.catchsha.outputs.GITHUB_SHA }}
+
+      - name: Deploy to Cloud Run
+        uses: actions-hub/gcloud@master
+        id: deploy
+        env:
+          PROJECT_ID: ${{ secrets.PROJECT_ID }}
+          DEV_PROJECT_ID: ${{ secrets.DEV_PROJECT_ID }}
+          GAR_LOCATION: ${{ secrets.GAR_LOCATION }}
+          GAR_NAME: ${{ secrets.GAR_NAME_API }}
+          SERVICE_NAME: '${{ secrets.SERVICE_NAME_API }}'
+          SERVICE_REGION: '${{ secrets.SERVICE_REGION_API }}'
+
+        with:
+          # args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:{{ steps.catchsha.outputs.GITHUB_SHA}} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' 
+      # Functionality not supported by Github Actions one to ccheck back agin in the future
+          args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.DEV_PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' 

Chart.yaml

@@ -0,0 +1 @@
+v0.0.0

src/config/server.js

@@ -236,6 +236,8 @@ app.use('/api/v2/questionbank', require('../resources/questionbank/questionbank.
 app.use('/api/v2/data-use-registers', require('../resources/dataUseRegister/dataUseRegister.route'));
 app.use('/api/v1/locations', require('../resources/spatialfilter/SpatialRouter'));
 
+app.use('/api/v1/metadata', require('../resources/metadata/metadata.route'));
+
 initialiseAuthentication(app);
 
 // launch our backend into a port

src/resources/auth/auth.route.js

@@ -25,7 +25,7 @@ router.get('/status', function (req, res, next) {
 		if (err || !user) {
 			return res.json({
 				success: true,
-				data: [{ role: 'Reader', id: null, name: null, loggedIn: false }],
+				data: [{ role: 'Reader', id: null, name: null, loggedIn: false, tempProp: true }],
 			});
 		} else {
 			// 1. Reformat teams array for frontend

src/resources/auth/utils.js

@@ -129,7 +129,7 @@ const getTeams = async () => {
 
 const catchLoginErrorAndRedirect = (req, res, next) => {
 	if (req.auth.err || !req.auth.user) {
-		if (req.auth.err === 'loginError') {
+		if (req.auth.err === 'loginError' || req.auth.user === undefined) {
 			return res.status(200).redirect(process.env.homeURL + '/loginerror');
 		}
 

src/resources/dataset/dataset.entity.js

@@ -50,7 +50,7 @@ export default class DatasetClass extends Entity {
 		transformedObject.dataset['@schema'] = {
 			type: `Dataset`,
 			version: `2.0.0`,
-			url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json`,
+			url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/2.1.0/dataset.schema.json`,
 		}
 
 		// Return v2 object

src/resources/metadata/metadata.route.js

@@ -0,0 +1,22 @@
+import express from 'express';
+import passport from 'passport';
+
+import { utils } from '../auth';
+import { ROLES } from '../user/user.roles';
+
+import datasetonboardingUtil from '../../utils/datasetonboarding.util';
+
+const router = express.Router({ mergeParams: true });
+
+router.post('/scoring', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin), async (req, res) => {
+    const { dataset } = req.body;
+
+    if (!dataset) {
+        res.json({ success: false, error: 'Dataset object must be supplied and contain all required data', status: 400 });
+    }
+
+    const verdict = await datasetonboardingUtil.buildMetadataQuality(dataset, dataset.datasetv2, dataset.pid);
+    res.json({ success: true, data: verdict, status: 200 });
+});
+
+module.exports = router;

src/utils/datasetonboarding.util.js

@@ -758,7 +758,7 @@ const buildMetadataQuality = async (dataset, v2Object, pid) => {
 	let rawdata = fs.readFileSync(__dirname + '/schema.json');
 	schema = JSON.parse(rawdata);
 
-	const ajv = new Ajv({ strict: false, allErrors: true });
+	const ajv = new Ajv({ strict: false, allErrors: false });
 	addFormats(ajv);
 	const validate = ajv.compile(schema);
 	validate(cleanV2Object);