From 190b4fcd9fbf87db43b07924474bd22fb61653cc Mon Sep 17 00:00:00 2001 From: brunoabreu-levio Date: Fri, 28 Jun 2024 12:48:19 -0400 Subject: [PATCH] Fixed the form flux --- .../iac/lambda.tf | 1 - .../iac/variables.tf | 7 ------ .../awsdemo/formrequestpreprocessor/App.java | 18 ++++++------- .../service/FormFillRequestDTO.java | 25 ++++++------------- .../events/event.json | 2 +- .../iac/lambda.tf | 13 +++++----- .../iac/variables.tf | 5 ++++ .../awsdemo/formrequestprocessor/App.java | 22 ++++++---------- .../service/FormFillRequestDTO.java | 11 ++++---- terraform/modules.tf | 2 +- 10 files changed, 45 insertions(+), 61 deletions(-) diff --git a/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/lambda.tf b/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/lambda.tf index d462170..63a427a 100644 --- a/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/lambda.tf +++ b/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/lambda.tf @@ -16,7 +16,6 @@ module "lambda_function_container_image" { environment_variables = { QUEUE_URL = var.queue_url - FORM_S3_URI = var.form_s3_uri } policy_statements = { diff --git a/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/variables.tf b/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/variables.tf index 9d65b97..d3498c3 100644 --- a/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/variables.tf +++ b/lambdas/FormProcessor/FormRequestPreProcessorFunction/iac/variables.tf @@ -27,10 +27,3 @@ variable "aws_region" { type = string default = "us-east-1" } - - -variable "form_s3_uri" { - type = string - nullable = false - default = "us-east-1" -} \ No newline at end of file diff --git a/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/App.java b/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/App.java index 6ebcee7..9ba7dc3 100644 --- a/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/App.java +++ b/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/App.java @@ -10,8 +10,6 @@ import com.levio.awsdemo.formrequestpreprocessor.service.SqsProducerService; public class App implements RequestHandler { - private final String formS3ObjectKey = System.getenv("FORM_S3_URI"); - private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JodaModule()); private final SqsProducerService sqsProducerService; @@ -33,13 +31,15 @@ public Void handleRequest(final S3EventNotification input, final Context context } input.getRecords().forEach(s3EventNotificationRecord -> { - var key = s3EventNotificationRecord.getS3().getObject().getKey(); - var formFillRequest = new FormFillRequestDTO(extractEmailId(key), extractFormKey(key), formS3ObjectKey); - System.out.print(formFillRequest); - sqsProducerService.send(formFillRequest); - } - - ); + var key = s3EventNotificationRecord.getS3().getObject().getKey(); + var bucketName = s3EventNotificationRecord.getS3().getBucket().getName(); + String emailId = extractEmailId(key); + var emailS3URI = "s3://" + bucketName + "/" + extractFormKey(key) + "/email/" + emailId; + var emailAttachmentS3URI = "s3://" + bucketName + "/" + key; + var formFillRequest = new FormFillRequestDTO(emailId, emailS3URI, emailAttachmentS3URI); + System.out.print(formFillRequest); + sqsProducerService.send(formFillRequest); + }); return null; } diff --git a/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/service/FormFillRequestDTO.java b/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/service/FormFillRequestDTO.java index 06f2ba9..132a75c 100644 --- a/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/service/FormFillRequestDTO.java +++ b/lambdas/FormProcessor/FormRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/formrequestpreprocessor/service/FormFillRequestDTO.java @@ -1,32 +1,23 @@ package com.levio.awsdemo.formrequestpreprocessor.service; public class FormFillRequestDTO { - /** - * The ID of the email which initiated the request - */ - String emailId; + private final String emailId; - /** - * The key to the form to be filled - */ - String formKey; + private final String emailS3URI; - /** - * The key to the attachment to parse - */ - String formS3URI; + private final String emailAttachmentS3URI; - public FormFillRequestDTO(String emailId, String formKey, String formS3ObjectKey) { + public FormFillRequestDTO(String emailId, String emailS3URI, String emailAttachmentS3URI) { this.emailId = emailId; - this.formKey = formKey; - this.formS3URI = formS3ObjectKey; + this.emailS3URI = emailS3URI; + this.emailAttachmentS3URI = emailAttachmentS3URI; } public String toJson() { return "{" + "\"emailId\":\"" + emailId + "\"," + - "\"formKey\":\"" + formKey + "\"," + - "\"formS3URI\":\"" + formS3URI + "\"" + + "\"emailS3URI\":\"" + emailS3URI + "\"," + + "\"emailAttachmentS3URI\":\"" + emailAttachmentS3URI + "\"" + "}"; } } diff --git a/lambdas/FormProcessor/FormRequestProcessorFunction/events/event.json b/lambdas/FormProcessor/FormRequestProcessorFunction/events/event.json index 1a920e7..ecad093 100644 --- a/lambdas/FormProcessor/FormRequestProcessorFunction/events/event.json +++ b/lambdas/FormProcessor/FormRequestProcessorFunction/events/event.json @@ -3,7 +3,7 @@ { "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", "receiptHandle": "MessageReceiptHandle", - "body": "u3a14tlds9k86f7374fuflhjgpjta1leq7a4cu01", + "body": "{\"emailId\":\"dmrhmr4s00t2s90jebn3djke8eh0tmon1u318ag1\",\"emailS3URI\":\"s3://levio-demo-fev-esta-ses-bucket-dev/formulaire/email/dmrhmr4s00t2s90jebn3djke8eh0tmon1u318ag1\",\"emailAttachmentS3URI\":\"s3://levio-demo-fev-esta-ses-bucket-dev/formulaire/attachment/dmrhmr4s00t2s90jebn3djke8eh0tmon1u318ag1.txt\"}", "attributes": { "sender": "bruno.abreu@levio.ca", "subject": "Levio HR" diff --git a/lambdas/FormProcessor/FormRequestProcessorFunction/iac/lambda.tf b/lambdas/FormProcessor/FormRequestProcessorFunction/iac/lambda.tf index 7f7ea9c..907a48c 100644 --- a/lambdas/FormProcessor/FormRequestProcessorFunction/iac/lambda.tf +++ b/lambdas/FormProcessor/FormRequestProcessorFunction/iac/lambda.tf @@ -18,11 +18,12 @@ module "lambda_function_container_image" { FUNCTION_NAME = var.resume_function_name QUEUE_URL = var.queue_url MASTER_PROMPT = var.master_prompt + FORM_S3_URI = var.form_s3_uri } policy_statements = { log_group = { - effect = "Allow" + effect = "Allow" actions = [ "logs:CreateLogGroup" ] @@ -32,7 +33,7 @@ module "lambda_function_container_image" { } log_write = { - effect = "Allow" + effect = "Allow" actions = [ "logs:CreateLogStream", "logs:PutLogEvents", @@ -43,7 +44,7 @@ module "lambda_function_container_image" { } s3 = { - effect = "Allow" + effect = "Allow" actions = [ "s3:Get*", "s3:List*", @@ -60,7 +61,7 @@ module "lambda_function_container_image" { } request_sqs = { - effect = "Allow" + effect = "Allow" actions = [ "sqs:ReceiveMessage", "sqs:DeleteMessage", @@ -72,7 +73,7 @@ module "lambda_function_container_image" { } response_sqs = { - effect = "Allow" + effect = "Allow" actions = [ "sqs:SendMessage", ] @@ -82,7 +83,7 @@ module "lambda_function_container_image" { } lambda = { - effect = "Allow" + effect = "Allow" actions = [ "lambda:InvokeFunction", ] diff --git a/lambdas/FormProcessor/FormRequestProcessorFunction/iac/variables.tf b/lambdas/FormProcessor/FormRequestProcessorFunction/iac/variables.tf index 95c1ee4..565b997 100644 --- a/lambdas/FormProcessor/FormRequestProcessorFunction/iac/variables.tf +++ b/lambdas/FormProcessor/FormRequestProcessorFunction/iac/variables.tf @@ -51,4 +51,9 @@ variable "sqs_name" { variable "aws_region" { type = string default = "us-east-1" +} + +variable "form_s3_uri" { + type = string + nullable = false } \ No newline at end of file diff --git a/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/App.java b/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/App.java index 5408db6..95cf8d9 100644 --- a/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/App.java +++ b/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/App.java @@ -19,6 +19,7 @@ import java.util.Map; public class App implements RequestHandler { + private static final String FORM_S3_URI = System.getenv("FORM_S3_URI"); private final DocumentService documentService; @@ -30,10 +31,8 @@ public class App implements RequestHandler { private final MailService mailService; - private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JodaModule()); - public App() { this.mailService = new MailService(); this.sqsProducerService = new SqsProducerService(); @@ -44,8 +43,7 @@ public App() { public App(S3Service s3Service, DocumentService documentService, - ClaudeService claudeService, SqsProducerService sqsProducerService, MailService mailService, - HashMap> questionsMapper) { + ClaudeService claudeService, SqsProducerService sqsProducerService, MailService mailService) { this.s3Service = s3Service; this.documentService = documentService; this.claudeService = claudeService; @@ -64,20 +62,16 @@ public Void handleRequest(final SQSEvent input, final Context context) { throw new RuntimeException(e); } - var emailId = formFillRequest.getEmailId(); - var formKey = formFillRequest.getFormKey(); - var formS3URI = formFillRequest.getFormS3URI(); - var questionsMapper = retrieveDocumentMapper(formS3URI); - - String email = s3Service.getFile(formKey + "/email/" + formFillRequest.getEmailId()); + String email = s3Service.getFile(formFillRequest.getEmailS3URI()); try { MimeMessage message = mailService.getMimeMessage(new ByteArrayInputStream(email.getBytes(StandardCharsets.UTF_8))); String emailBody = "Formulaire response"; String sender = ((InternetAddress) message.getFrom()[0]).getAddress(); String subject = message.getSubject(); - String content = s3Service.getFile(formS3URI); + String content = s3Service.getFile(formFillRequest.getEmailAttachmentS3URI()); + var questionsMapper = retrieveDocumentMapper(FORM_S3_URI); questionsMapper.entrySet().parallelStream() .forEach(positionQuestionAnswerMapper -> { Map questionAnswerMap = positionQuestionAnswerMapper.getValue(); @@ -85,9 +79,9 @@ public Void handleRequest(final SQSEvent input, final Context context) { String answer = claudeService.getResponse(questionAnswerMap.get("question"), content); questionAnswerMap.put("answer", answer); }); - ByteArrayOutputStream fileOutputStream = documentService.fillFile(questionsMapper, formS3URI); - String formDocxUri = s3Service.saveFile(formKey+"/" + emailId + ".docx", fileOutputStream.toByteArray()); - sqsProducerService.send(emailBody, getMessageAttributes(sender, subject, formDocxUri), emailId); + ByteArrayOutputStream fileOutputStream = documentService.fillFile(questionsMapper, FORM_S3_URI); + String formDocxUri = s3Service.saveFile("formulaire/" + formFillRequest.getEmailId() + ".docx", fileOutputStream.toByteArray()); + sqsProducerService.send(emailBody, getMessageAttributes(sender, subject, formDocxUri), formFillRequest.getEmailId()); } catch (IOException | MessagingException e) { throw new RuntimeException(e); } diff --git a/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/service/FormFillRequestDTO.java b/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/service/FormFillRequestDTO.java index 11b4236..7971e4d 100644 --- a/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/service/FormFillRequestDTO.java +++ b/lambdas/FormProcessor/FormRequestProcessorFunction/src/main/java/com/levio/awsdemo/formrequestprocessor/service/FormFillRequestDTO.java @@ -9,12 +9,13 @@ @NoArgsConstructor @AllArgsConstructor public class FormFillRequestDTO { - @JsonProperty("emailId") // Map JSON property to field + @JsonProperty("emailId") private String emailId; - @JsonProperty("formKey") // Map JSON property to field - private String formKey; + @JsonProperty("emailS3URI") + private String emailS3URI; + + @JsonProperty("emailAttachmentS3URI") + private String emailAttachmentS3URI; - @JsonProperty("formS3URI") - private String formS3URI; } diff --git a/terraform/modules.tf b/terraform/modules.tf index 1253133..d950469 100644 --- a/terraform/modules.tf +++ b/terraform/modules.tf @@ -212,6 +212,7 @@ module "form_request_processor" { queue_url = module.email_response_processor.queue_url master_prompt = var.master_prompt sqs_name = local.form_request_processor_queue_name + form_s3_uri = local.form_s3_uri } module "form_request_preprocessor" { @@ -221,7 +222,6 @@ module "form_request_preprocessor" { ses_bucket_arn = module.s3_bucket.s3_bucket_arn request_queue_arn = module.form_request_processor.queue_arn queue_url = module.form_request_processor.queue_url - form_s3_uri = local.form_s3_uri } module "email_receipt_confirmation" {