Skip to content

Commit

Permalink
feat: add send events and API endpoint for uploading submissions to S3 (
Browse files Browse the repository at this point in the history
  • Loading branch information
jessicamcinchak authored Apr 11, 2024
1 parent 2e81ec2 commit 18ab668
Show file tree
Hide file tree
Showing 10 changed files with 207 additions and 59 deletions.
1 change: 1 addition & 0 deletions api.planx.uk/lib/hasura/metadata/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export interface CombinedResponse {
bops?: ScheduledEventResponse;
uniform?: ScheduledEventResponse;
email?: ScheduledEventResponse;
s3?: ScheduledEventResponse;
}

interface ScheduledEventArgs {
Expand Down
13 changes: 8 additions & 5 deletions api.planx.uk/modules/file/service/uploadFile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ const nanoid = customAlphabet("1234567890abcdefghijklmnopqrstuvwxyz", 8);
export const uploadPublicFile = async (
file: Express.Multer.File,
filename: string,
filekey?: string,
) => {
const s3 = s3Factory();

const { params, key, fileType } = generateFileParams(file, filename);
const { params, key, fileType } = generateFileParams(file, filename, filekey);

await s3.putObject(params).promise();
const fileUrl = buildFileUrl(key, "public");
Expand All @@ -25,10 +26,11 @@ export const uploadPublicFile = async (
export const uploadPrivateFile = async (
file: Express.Multer.File,
filename: string,
filekey?: string,
) => {
const s3 = s3Factory();

const { params, key, fileType } = generateFileParams(file, filename);
const { params, key, fileType } = generateFileParams(file, filename, filekey);

params.Metadata = {
is_private: "true",
Expand Down Expand Up @@ -59,20 +61,21 @@ const buildFileUrl = (key: string, path: "public" | "private") => {
export function generateFileParams(
file: Express.Multer.File,
filename: string,
filekey?: string,
): {
params: S3.PutObjectRequest;
fileType: string | null;
key: string;
} {
const fileType = getType(filename);
const key = `${nanoid()}/${filename}`;
const key = `${filekey || nanoid()}/${filename}`;

const params = {
ACL: process.env.AWS_S3_ACL,
Key: key,
Body: file.buffer,
Body: file?.buffer || JSON.stringify(file),
ContentDisposition: `inline;filename="${filename}"`,
ContentType: file.mimetype,
ContentType: file?.mimetype || "application/json",
} as S3.PutObjectRequest;

return {
Expand Down
12 changes: 11 additions & 1 deletion api.planx.uk/modules/send/createSendEvents/controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ const createSendEvents: CreateSendEventsController = async (
res,
next,
) => {
const { email, uniform, bops } = res.locals.parsedReq.body;
const { email, uniform, bops, s3 } = res.locals.parsedReq.body;
const { sessionId } = res.locals.parsedReq.params;

try {
Expand Down Expand Up @@ -47,6 +47,16 @@ const createSendEvents: CreateSendEventsController = async (
combinedResponse["uniform"] = uniformEvent;
}

if (s3) {
const s3Event = await createScheduledEvent({
webhook: `{{HASURA_PLANX_API_URL}}/upload-submission/${s3.localAuthority}`,
schedule_at: now,
payload: s3.body,
comment: `upload_submission_${sessionId}`,
});
combinedResponse["s3"] = s3Event;
}

return res.json(combinedResponse);
} catch (error) {
return next({
Expand Down
1 change: 1 addition & 0 deletions api.planx.uk/modules/send/createSendEvents/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ export const combinedEventsPayloadSchema = z.object({
email: eventSchema.optional(),
bops: eventSchema.optional(),
uniform: eventSchema.optional(),
s3: eventSchema.optional(),
}),
params: z.object({
sessionId: z.string().uuid(),
Expand Down
2 changes: 2 additions & 0 deletions api.planx.uk/modules/send/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { sendToEmail } from "./email";
import { validate } from "../../shared/middleware/validate";
import { combinedEventsPayloadSchema } from "./createSendEvents/types";
import { downloadApplicationFiles } from "./downloadApplicationFiles";
import { sendToS3 } from "./s3";

const router = Router();

Expand All @@ -19,5 +20,6 @@ router.post("/bops/:localAuthority", useHasuraAuth, sendToBOPS);
router.post("/uniform/:localAuthority", useHasuraAuth, sendToUniform);
router.post("/email-submission/:localAuthority", useHasuraAuth, sendToEmail);
router.get("/download-application-files/:sessionId", downloadApplicationFiles);
router.post("/upload-submission/:localAuthority", useHasuraAuth, sendToS3);

export default router;
60 changes: 60 additions & 0 deletions api.planx.uk/modules/send/s3/index.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import supertest from "supertest";
import app from "../../../server";
import { expectedPlanningPermissionPayload } from "../../../tests/mocks/digitalPlanningDataMocks";

jest.mock("../../saveAndReturn/service/utils", () => ({
markSessionAsSubmitted: jest.fn(),
}));

jest.mock("@opensystemslab/planx-core", () => {
const actualCoreDomainClient = jest.requireActual(
"@opensystemslab/planx-core",
).CoreDomainClient;

return {
CoreDomainClient: class extends actualCoreDomainClient {
constructor() {
super();
this.export.digitalPlanningDataPayload = () =>
jest.fn().mockResolvedValue({
exportData: expectedPlanningPermissionPayload,
});
}
},
};
});

describe(`uploading an application to S3`, () => {
it("requires auth", async () => {
await supertest(app)
.post("/upload-submission/barnet")
.send({ payload: { sessionId: "123" } })
.expect(401);
});

it("throws an error if payload is missing", async () => {
await supertest(app)
.post("/upload-submission/barnet")
.set({ Authorization: process.env.HASURA_PLANX_API_KEY! })
.send({ payload: null })
.expect(400)
.then((res) => {
expect(res.body.error).toMatch(/Missing application payload/);
});
});

it("throws an error if team is unsupported", async () => {
await supertest(app)
.post("/upload-submission/unsupported-team")
.set({ Authorization: process.env.HASURA_PLANX_API_KEY! })
.send({ payload: { sessionId: "123" } })
.expect(400)
.then((res) => {
expect(res.body.error).toMatch(
"Send to S3 is not enabled for this local authority (unsupported-team)",
);
});
});

it.todo("succeeds"); // mock uploadPrivateFile ??
});
60 changes: 60 additions & 0 deletions api.planx.uk/modules/send/s3/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import type { NextFunction, Request, Response } from "express";
import { $api } from "../../../client";
import { uploadPrivateFile } from "../../file/service/uploadFile";
import { markSessionAsSubmitted } from "../../saveAndReturn/service/utils";

export async function sendToS3(
req: Request,
res: Response,
next: NextFunction,
) {
// `/upload-submission/:localAuthority` is only called via Hasura's scheduled event webhook, so body is wrapped in a "payload" key
const { payload } = req.body;
const localAuthority = req.params.localAuthority;

if (!payload?.sessionId) {
return next({
status: 400,
message: `Missing application payload data to send to email`,
});
}

try {
const { sessionId } = payload;

// Only prototyping with Barnet to begin
// In future, confirm this local authority has an S3 bucket/folder configured in team_integrations or similar
if (localAuthority !== "barnet") {
return next({
status: 400,
message: `Send to S3 is not enabled for this local authority (${localAuthority})`,
});
}

// Generate the ODP Schema JSON
const exportData = await $api.export.digitalPlanningDataPayload(sessionId);

// Create and upload the data as an S3 file
const { fileUrl } = await uploadPrivateFile(
exportData,
`${sessionId}.json`,
"barnet-prototype",
);

// Mark session as submitted so that reminder and expiry emails are not triggered
markSessionAsSubmitted(sessionId);

// TODO Create and update an audit table entry

return res.status(200).send({
message: `Successfully uploaded submission to S3: ${fileUrl}`,
});
} catch (error) {
return next({
error,
message: `Failed to upload submission to S3 (${localAuthority}): ${
(error as Error).message
}`,
});
}
}
106 changes: 55 additions & 51 deletions editor.planx.uk/src/@planx/components/Send/Editor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -65,62 +65,66 @@ const SendComponent: React.FC<Props> = (props) => {
});
}

const changeCheckbox = (value: Destination) => (_checked: any) => {
let newCheckedValues: Destination[];
const changeCheckbox =
(value: Destination) =>
(_checked: React.MouseEvent<HTMLButtonElement, MouseEvent> | undefined) => {
let newCheckedValues: Destination[];

if (formik.values.destinations.includes(value)) {
newCheckedValues = formik.values.destinations.filter((x) => x !== value);
} else {
newCheckedValues = [...formik.values.destinations, value];
}

formik.setFieldValue(
"destinations",
newCheckedValues.sort((a, b) => {
const originalValues = options.map((cb) => cb.value);
return originalValues.indexOf(a) - originalValues.indexOf(b);
}),
);
if (formik.values.destinations.includes(value)) {
newCheckedValues = formik.values.destinations.filter(
(x) => x !== value,
);
} else {
newCheckedValues = [...formik.values.destinations, value];
}

// Show warnings on selection of BOPS or Uniform for likely unsupported services
// Don't actually restrict selection because flowSlug matching is imperfect for some valid test cases
const teamSlug = window.location.pathname?.split("/")?.[1];
const flowSlug = window.location.pathname?.split("/")?.[2];
if (
value === Destination.BOPS &&
newCheckedValues.includes(value) &&
![
"apply-for-a-lawful-development-certificate",
"apply-for-prior-approval",
"apply-for-planning-permission",
].includes(flowSlug)
) {
alert(
"BOPS only accepts Lawful Development Certificate, Prior Approval, and Planning Permission submissions. Please do not select if you're building another type of submission service!",
formik.setFieldValue(
"destinations",
newCheckedValues.sort((a, b) => {
const originalValues = options.map((cb) => cb.value);
return originalValues.indexOf(a) - originalValues.indexOf(b);
}),
);
}

if (
value === Destination.Uniform &&
newCheckedValues.includes(value) &&
flowSlug !== "apply-for-a-lawful-development-certificate" &&
!["buckinghamshire", "lambeth", "southwark"].includes(teamSlug)
) {
alert(
"Uniform is only enabled for Bucks, Lambeth and Southwark to accept Lawful Development Certificate submissions. Please do not select if you're building another type of submission service!",
);
}
// Show warnings on selection of BOPS or Uniform for likely unsupported services
// Don't actually restrict selection because flowSlug matching is imperfect for some valid test cases
const teamSlug = window.location.pathname?.split("/")?.[1];
const flowSlug = window.location.pathname?.split("/")?.[2];
if (
value === Destination.BOPS &&
newCheckedValues.includes(value) &&
![
"apply-for-a-lawful-development-certificate",
"apply-for-prior-approval",
"apply-for-planning-permission",
].includes(flowSlug)
) {
alert(
"BOPS only accepts Lawful Development Certificate, Prior Approval, and Planning Permission submissions. Please do not select if you're building another type of submission service!",
);
}

if (
value === Destination.S3 &&
newCheckedValues.includes(value) &&
teamSlug !== "barnet"
) {
alert(
"AWS S3 uploads are currently being prototyped with Barnet only. Please do not select this option for other councils yet.",
);
}
};
if (
value === Destination.Uniform &&
newCheckedValues.includes(value) &&
flowSlug !== "apply-for-a-lawful-development-certificate" &&
!["buckinghamshire", "lambeth", "southwark"].includes(teamSlug)
) {
alert(
"Uniform is only enabled for Bucks, Lambeth and Southwark to accept Lawful Development Certificate submissions. Please do not select if you're building another type of submission service!",
);
}

if (
value === Destination.S3 &&
newCheckedValues.includes(value) &&
teamSlug !== "barnet"
) {
alert(
"AWS S3 uploads are currently being prototyped with Barnet only. Please do not select this option for other councils yet.",
);
}
};

return (
<form onSubmit={formik.handleSubmit} id="modal">
Expand Down
2 changes: 1 addition & 1 deletion editor.planx.uk/src/@planx/components/Send/Public.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ const SendComponent: React.FC<Props> = ({
makeData(props, request.value.s3?.event_id, "s3SendEventId"),
);
}
}, [request.loading, request.error, request.value]);
}, [request.loading, request.error, request.value, destinations, props]);

if (request.loading) {
return (
Expand Down
9 changes: 8 additions & 1 deletion editor.planx.uk/src/@planx/components/Send/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,13 @@ export enum Destination {
S3 = "s3",
}

interface EventPayload {
localAuthority: string;
body: {
sessionId: string;
};
}

export interface Send extends MoreInformation {
title: string;
destinations: Destination[];
Expand All @@ -33,7 +40,7 @@ export function getCombinedEventsPayload({
passport: Store.passport;
sessionId: string;
}) {
const combinedEventsPayload: any = {};
const combinedEventsPayload: Record<string, EventPayload> = {};

// Format application user data as required by BOPS
if (destinations.includes(Destination.BOPS)) {
Expand Down

0 comments on commit 18ab668

Please sign in to comment.