diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c6243b2..25f7381 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,6 +37,11 @@ jobs: aws ecr describe-repositories --repository-names ${{ vars.EMAIL_REQUEST_PROCESSOR_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.EMAIL_REQUEST_PROCESSOR_ECR_NAME }} aws ecr describe-repositories --repository-names ${{ vars.EMAIL_REQUEST_PREPROCESSOR_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.EMAIL_REQUEST_PREPROCESSOR_ECR_NAME }} aws ecr describe-repositories --repository-names ${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }} + aws ecr describe-repositories --repository-names ${{ vars.ATTACHMENT_SAVER_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.ATTACHMENT_SAVER_ECR_NAME }} + aws ecr describe-repositories --repository-names ${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }} + aws ecr describe-repositories --repository-names ${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }} + aws ecr describe-repositories --repository-names ${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }} + aws ecr describe-repositories --repository-names ${{ vars.RESUME_ECR_NAME }} || aws ecr create-repository --repository-name ${{ vars.RESUME_ECR_NAME }} - name: Generate timestamp id: timestamp @@ -64,6 +69,16 @@ jobs: docker push $ECR_REGISTRY/${{ vars.EMAIL_REQUEST_PREPROCESSOR_ECR_NAME }}:$IMAGE_TAG docker build -t $ECR_REGISTRY/${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }}:$IMAGE_TAG ./lambdas/EmailProcessor/EmailResponseProcessorFunction docker push $ECR_REGISTRY/${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }}:$IMAGE_TAG + docker build -t $ECR_REGISTRY/${{ vars.ATTACHMENT_SAVER_ECR_NAME }}:$IMAGE_TAG ./lambdas/AttachmentSaver/AttachmentSaverFunction + docker push $ECR_REGISTRY/${{ vars.ATTACHMENT_SAVER_ECR_NAME }}:$IMAGE_TAG + docker build -t $ECR_REGISTRY/${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }}:$IMAGE_TAG ./lambdas/TranscriptionProcessor/TranscriptionFunction + docker push $ECR_REGISTRY/${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }}:$IMAGE_TAG + docker build -t $ECR_REGISTRY/${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }}:$IMAGE_TAG ./lambdas/ResumeProcessor/ResumeRequestProcessorFunction + docker push $ECR_REGISTRY/${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }}:$IMAGE_TAG + docker build -t $ECR_REGISTRY/${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }}:$IMAGE_TAG ./lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction + docker push $ECR_REGISTRY/${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }}:$IMAGE_TAG + docker build -t $ECR_REGISTRY/${{ vars.RESUME_ECR_NAME }}:$IMAGE_TAG ./lambdas/ResumeProcessor/ResumeFunction + docker push $ECR_REGISTRY/${{ vars.RESUME_ECR_NAME }}:$IMAGE_TAG terraform_plan: if: github.event_name == 'pull_request' @@ -112,6 +127,11 @@ jobs: -var="email_response_processor_lambda_repository_name=${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }}" \ -var="sender_email=${{ vars.SENDER_EMAIL }}" \ -var="chat_rule_recipient=${{ vars.CHAT_RULE_RECIPIENT }}" \ + -var="attachment_saver_lambda_repository_name=${{ vars.ATTACHMENT_SAVER_ECR_NAME }}" \ + -var="transcription_processor_lambda_repository_name=${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }}" \ + -var="resume_request_processor_lambda_repository_name=${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }}" \ + -var="resume_request_preprocessor_lambda_repository_name=${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }}" \ + -var="resume_lambda_repository_name=${{ vars.RESUME_ECR_NAME }}" \ -no-color -out=terraform.tfplan - name: Upload Terraform Plan @@ -200,6 +220,11 @@ jobs: for digest in $(aws ecr list-images --repository-name ${{ vars.EMAIL_REQUEST_PROCESSOR_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.EMAIL_REQUEST_PROCESSOR_ECR_NAME }} --image-ids imageDigest=$digest; done for digest in $(aws ecr list-images --repository-name ${{ vars.EMAIL_REQUEST_PREPROCESSOR_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.EMAIL_REQUEST_PREPROCESSOR_ECR_NAME }} --image-ids imageDigest=$digest; done for digest in $(aws ecr list-images --repository-name ${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }} --image-ids imageDigest=$digest; done + for digest in $(aws ecr list-images --repository-name ${{ vars.ATTACHMENT_SAVER_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.ATTACHMENT_SAVER_ECR_NAME }} --image-ids imageDigest=$digest; done + for digest in $(aws ecr list-images --repository-name ${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }} --image-ids imageDigest=$digest; done + for digest in $(aws ecr list-images --repository-name ${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }} --image-ids imageDigest=$digest; done + for digest in $(aws ecr list-images --repository-name ${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }} --image-ids imageDigest=$digest; done + for digest in $(aws ecr list-images --repository-name ${{ vars.RESUME_ECR_NAME }} --query 'sort_by(imageIds,&imagePushedAt)[0:-2].imageDigest' --output text); do aws ecr batch-delete-image --repository-name ${{ vars.RESUME_ECR_NAME }} --image-ids imageDigest=$digest; done deploy-infra: if: github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -236,6 +261,11 @@ jobs: -var="email_response_processor_lambda_repository_name=${{ vars.EMAIL_RESPONSE_PROCESSOR_ECR_NAME }}" \ -var="sender_email=${{ vars.SENDER_EMAIL }}" \ -var="chat_rule_recipient=${{ vars.CHAT_RULE_RECIPIENT }}" \ + -var="attachment_saver_lambda_repository_name=${{ vars.ATTACHMENT_SAVER_ECR_NAME }}" \ + -var="transcription_processor_lambda_repository_name=${{ vars.TRANSCRIPTION_PROCESSOR_ECR_NAME }}" \ + -var="resume_request_processor_lambda_repository_name=${{ vars.RESUME_REQUEST_PROCESSOR_ECR_NAME }}" \ + -var="resume_request_preprocessor_lambda_repository_name=${{ vars.RESUME_REQUEST_PREPROCESSOR_ECR_NAME }}" \ + -var="resume_lambda_repository_name=${{ vars.RESUME_ECR_NAME }}" \ -auto-approve -no-color -input=false diff --git a/lambdas/AttachmentSaver/AttachmentSaver.iml b/lambdas/AttachmentSaver/AttachmentSaver.iml new file mode 100644 index 0000000..482334b --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaver.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/Dockerfile b/lambdas/AttachmentSaver/AttachmentSaverFunction/Dockerfile new file mode 100644 index 0000000..73a72f7 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/Dockerfile @@ -0,0 +1,15 @@ +FROM public.ecr.aws/sam/build-java17:latest as build-image + +WORKDIR "/task" +COPY src/ src/ +COPY pom.xml ./ + +RUN mvn -q clean install +RUN mvn dependency:copy-dependencies -DincludeScope=compile + +FROM public.ecr.aws/lambda/java:17 + +COPY --from=build-image /task/target/classes /var/task/ +COPY --from=build-image /task/target/dependency /var/task/lib +# Command can be overwritten by providing a different command in the template directly. +CMD ["com.levio.awsdemo.attachmentsaver.App::handleRequest"] \ No newline at end of file diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/dependency-reduced-pom.xml b/lambdas/AttachmentSaver/AttachmentSaverFunction/dependency-reduced-pom.xml new file mode 100644 index 0000000..33b6018 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/dependency-reduced-pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + com.levio.aws-demo + attachment-saver + Attachment Saver + 1.0 + + + + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + + + + + + junit + junit + 4.13.2 + test + + + hamcrest-core + org.hamcrest + + + + + + 17 + 17 + + diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/events/event.json b/lambdas/AttachmentSaver/AttachmentSaverFunction/events/event.json new file mode 100644 index 0000000..995c40b --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/events/event.json @@ -0,0 +1,38 @@ +{ + "Records": [ + { + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": { + "principalId": "EXAMPLE" + }, + "requestParameters": { + "sourceIPAddress": "127.0.0.1" + }, + "responseElements": { + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": { + "name": "levio-demo-fev-esta-ses-bucket-dev", + "ownerIdentity": { + "principalId": "EXAMPLE" + }, + "arn": "arn:aws:s3:::example-bucket" + }, + "object": { + "key": "resume/email/u987v5umtglfcd0gm92traijlp7vqq44g8uat401", + "size": 1024, + "eTag": "0123456789abcdef0123456789abcdef", + "sequencer": "0A1B2C3D4E5F678901" + } + } + } + ] +} \ No newline at end of file diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/lambda.tf b/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/lambda.tf new file mode 100644 index 0000000..4fa2488 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/lambda.tf @@ -0,0 +1,71 @@ +data "aws_ecr_image" "lambda_image" { + repository_name = var.lambda_repository_name + most_recent = true +} + +data "aws_caller_identity" "current" {} + +module "lambda_function_container_image" { + timeout = 60 + source = "terraform-aws-modules/lambda/aws" + function_name = var.lambda_function_name + create_package = false + image_uri = data.aws_ecr_image.lambda_image.image_uri + package_type = "Image" + memory_size = 1024 + role_name = "${var.lambda_function_name}-role" + attach_policy_statements = true + + environment_variables = { + } + + policy_statements = { + log_group = { + effect = "Allow" + actions = [ + "logs:CreateLogGroup" + ] + resources = [ + "arn:aws:logs:*:*:*" + ] + } + + log_write = { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [ + "arn:aws:logs:*:*:log-group:/aws/${var.lambda_function_name}/*:*" + ] + } + + s3 = { + effect = "Allow" + actions = [ + "s3:Get*", + "s3:List*", + "s3:Describe*", + "s3:PutObject", + "s3-object-lambda:Get*", + "s3-object-lambda:List*", + "s3-object-lambda:WriteGetObjectResponse" + ] + resources = [ + var.ses_bucket_arn, + "${var.ses_bucket_arn}/*" + ] + } + } + + create_current_version_allowed_triggers = false + + allowed_triggers = { + s3 = { + principal = "s3.amazonaws.com" + source_arn = var.ses_bucket_arn + } + } + +} \ No newline at end of file diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/output.tf b/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/output.tf new file mode 100644 index 0000000..9e15297 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/output.tf @@ -0,0 +1,3 @@ +output "lambda_function_arn" { + value = module.lambda_function_container_image.lambda_function_arn +} \ No newline at end of file diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/variables.tf b/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/variables.tf new file mode 100644 index 0000000..06e90d8 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/iac/variables.tf @@ -0,0 +1,24 @@ +variable "lambda_function_name" { + type = string + nullable = false +} + +variable "lambda_repository_name" { + type = string + nullable = false +} + +variable "ses_bucket_name" { + type = string + nullable = false +} + +variable "ses_bucket_arn" { + type = string + nullable = false +} + +variable "aws_region" { + type = string + default = "us-east-1" +} \ No newline at end of file diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/pom.xml b/lambdas/AttachmentSaver/AttachmentSaverFunction/pom.xml new file mode 100644 index 0000000..174cdec --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/pom.xml @@ -0,0 +1,82 @@ + + 4.0.0 + com.levio.aws-demo + attachment-saver + 1.0 + jar + Attachment Saver + + 17 + 17 + + + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-lambda-java-events + 3.11.0 + + + software.amazon.awssdk + s3 + 2.20.52 + + + software.amazon.awssdk + apache-client + 2.20.52 + + + com.sun.mail + jakarta.mail + 2.0.1 + + + jakarta.mail + jakarta.mail-api + 2.1.2 + + + com.fasterxml.jackson.core + jackson-databind + 2.15.3 + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + 2.15.3 + + + junit + junit + 4.13.2 + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + + + package + + shade + + + + + + + diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/App.java b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/App.java new file mode 100644 index 0000000..aa6aed0 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/App.java @@ -0,0 +1,55 @@ +package com.levio.awsdemo.attachmentsaver; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.models.s3.S3EventNotification; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.joda.JodaModule; +import com.levio.awsdemo.attachmentsaver.service.S3Service; +import jakarta.mail.MessagingException; +import jakarta.mail.Part; + +import java.io.IOException; +import java.util.List; + +public class App implements RequestHandler { + + private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JodaModule()); + + private final S3Service s3Service; + + public App() { + this.s3Service = new S3Service(); + } + + public App(S3Service s3Service) { + this.s3Service = s3Service; + } + + public Void handleRequest(final S3EventNotification input, final Context context) { + try { + String json = objectMapper.writeValueAsString(input); + System.out.println(json); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + input.getRecords().forEach(s3EventNotificationRecord -> { + try { + List attachments = s3Service.getAttachments(s3EventNotificationRecord.getS3()); + attachments.forEach(part -> { + try { + s3Service.saveAttachment(part, s3EventNotificationRecord.getS3()); + } catch (MessagingException | IOException e) { + throw new RuntimeException(e); + } + }); + } catch (MessagingException | IOException e) { + throw new RuntimeException(e); + } + }); + + return null; + } +} diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/service/S3Service.java b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/service/S3Service.java new file mode 100644 index 0000000..6c5d926 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/service/S3Service.java @@ -0,0 +1,85 @@ +package com.levio.awsdemo.attachmentsaver.service; + +import com.amazonaws.services.lambda.runtime.events.models.s3.S3EventNotification; +import com.levio.awsdemo.attachmentsaver.util.EmailUtils; +import jakarta.mail.MessagingException; +import jakarta.mail.Part; +import jakarta.mail.Session; +import jakarta.mail.internet.MimeMessage; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Properties; + +public class S3Service { + + private final S3Client s3 = S3Client.builder() + .region(Region.US_EAST_1) + .build(); + + public List getAttachments(S3EventNotification.S3Entity s3Entity) throws MessagingException, IOException { + GetObjectRequest objectRequest = GetObjectRequest + .builder() + .key(s3Entity.getObject().getKey()) + .bucket(s3Entity.getBucket().getName()) + .build(); + + ResponseBytes objectBytes = s3.getObjectAsBytes(objectRequest); + System.out.println(new String(objectBytes.asByteArray())); + InputStream inputStream = objectBytes.asInputStream(); + + Properties props = new Properties(); + Session session = Session.getDefaultInstance(props, null); + + MimeMessage message = new MimeMessage(session, inputStream); + + return EmailUtils.extractAttachments(message); + } + + public void saveAttachment(Part part, S3EventNotification.S3Entity s3Entity) throws MessagingException, IOException { + try (InputStream inputStream = part.getInputStream()) { + String key = getKey(part, s3Entity); + PutObjectResponse objectResponse = s3.putObject( + PutObjectRequest.builder() + .bucket(s3Entity.getBucket().getName()) + .key(key) + .build(), + RequestBody.fromBytes(inputStream.readAllBytes())); + if (objectResponse.sdkHttpResponse().isSuccessful()) { + System.out.println("File " + key + " created" ); + } + } + } + + private String getKey(Part part, S3EventNotification.S3Entity s3Entity) throws MessagingException { + return extractKeyPath(s3Entity.getObject().getKey()) + + "attachment/" + + getEmailId(s3Entity.getObject().getKey()) + + extractFileExtension(part.getFileName()); + } + + private String extractFileExtension(String fileName) { + return "." + fileName.substring(fileName.lastIndexOf(".") + 1); + } + + private String extractKeyPath(String path) { + int lastIndex = path.lastIndexOf("/"); + lastIndex = path.substring(0, lastIndex).lastIndexOf("/"); + return path.substring(0, lastIndex) + "/"; + } + + private String getEmailId(String path) { + String[] parts = path.split("/"); + return parts[parts.length - 1]; + } + +} diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/util/EmailUtils.java b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/util/EmailUtils.java new file mode 100644 index 0000000..d16d296 --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/main/java/com/levio/awsdemo/attachmentsaver/util/EmailUtils.java @@ -0,0 +1,28 @@ +package com.levio.awsdemo.attachmentsaver.util; + +import jakarta.mail.MessagingException; +import jakarta.mail.Multipart; +import jakarta.mail.Part; +import jakarta.mail.internet.MimeMessage; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class EmailUtils { + public static List extractAttachments(MimeMessage message) throws MessagingException, IOException { + Object content = message.getContent(); + + List attachments = new ArrayList<>(); + + if (content instanceof Multipart multipart) { + for (int i = 0; i < multipart.getCount(); i++) { + Part part = multipart.getBodyPart(i); + if (Part.ATTACHMENT.equalsIgnoreCase(part.getDisposition())) { + attachments.add(part); + } + } + } + return attachments; + } +} diff --git a/lambdas/AttachmentSaver/AttachmentSaverFunction/src/test/java/com/levio/awsdemo/attachmentsaver/AppTest.java b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/test/java/com/levio/awsdemo/attachmentsaver/AppTest.java new file mode 100644 index 0000000..cbee63c --- /dev/null +++ b/lambdas/AttachmentSaver/AttachmentSaverFunction/src/test/java/com/levio/awsdemo/attachmentsaver/AppTest.java @@ -0,0 +1,22 @@ +//package com.levio.awsdemo.attachmentsaver; +// +//import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +//import static org.junit.Assert.assertEquals; +//import static org.junit.Assert.assertNotNull; +//import static org.junit.Assert.assertTrue; +//import org.junit.Test; +// +//public class AppTest { +// @Test +// public void successfulResponse() { +// App app = new App(); +// APIGatewayProxyResponseEvent result = app.handleRequest(null, null); +// assertEquals(200, result.getStatusCode().intValue()); +// assertEquals("application/json", result.getHeaders().get("Content-Type")); +// String content = result.getBody(); +// assertNotNull(content); +// assertTrue(content.contains("\"message\"")); +// assertTrue(content.contains("\"hello world\"")); +// assertTrue(content.contains("\"location\"")); +// } +//} diff --git a/lambdas/ResumeProcessor/ResumeFunction/Dockerfile b/lambdas/ResumeProcessor/ResumeFunction/Dockerfile new file mode 100644 index 0000000..81eb8ba --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/Dockerfile @@ -0,0 +1,15 @@ +FROM public.ecr.aws/sam/build-java17:latest as build-image + +WORKDIR "/task" +COPY src/ src/ +COPY pom.xml ./ + +RUN mvn -q clean install +RUN mvn dependency:copy-dependencies -DincludeScope=compile + +FROM public.ecr.aws/lambda/java:17 + +COPY --from=build-image /task/target/classes /var/task/ +COPY --from=build-image /task/target/dependency /var/task/lib +# Command can be overwritten by providing a different command in the template directly. +CMD ["com.levio.awsdemo.resume.App::handleRequest"] \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeFunction/dependency-reduced-pom.xml b/lambdas/ResumeProcessor/ResumeFunction/dependency-reduced-pom.xml new file mode 100644 index 0000000..f6e2612 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/dependency-reduced-pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + com.levio.aws-demo + resume + Resume + 1.0 + + + + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + + + + + + junit + junit + 4.13.2 + test + + + hamcrest-core + org.hamcrest + + + + + + 17 + 17 + + diff --git a/lambdas/ResumeProcessor/ResumeFunction/events/event.json b/lambdas/ResumeProcessor/ResumeFunction/events/event.json new file mode 100644 index 0000000..6f59884 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/events/event.json @@ -0,0 +1,4 @@ +{ + "prompt": "Refais ce texte sous forme de dialogues entre un intervenant et ses clients", + "text": "value2" +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeFunction/iac/lambda.tf b/lambdas/ResumeProcessor/ResumeFunction/iac/lambda.tf new file mode 100644 index 0000000..2bf193a --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/iac/lambda.tf @@ -0,0 +1,56 @@ +data "aws_ecr_image" "lambda_image" { + repository_name = var.lambda_repository_name + most_recent = true +} + +data "aws_caller_identity" "current" {} + +module "lambda_function_container_image" { + timeout = 60 + source = "terraform-aws-modules/lambda/aws" + function_name = var.lambda_function_name + create_package = false + image_uri = data.aws_ecr_image.lambda_image.image_uri + package_type = "Image" + memory_size = 1024 + role_name = "${var.lambda_function_name}-role" + attach_policy_statements = true + + environment_variables = { + PROMPT = var.prompt_default + } + + policy_statements = { + log_group = { + effect = "Allow" + actions = [ + "logs:CreateLogGroup" + ] + resources = [ + "arn:aws:logs:*:*:*" + ] + } + + log_write = { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [ + "arn:aws:logs:*:*:log-group:/aws/${var.lambda_function_name}/*:*" + ] + } + + bedrock = { + effect = "Allow" + actions = [ + "bedrock:InvokeModel" + ] + resources = [ + "arn:aws:bedrock:*:*:*" + ] + } + + } +} diff --git a/lambdas/ResumeProcessor/ResumeFunction/iac/output.tf b/lambdas/ResumeProcessor/ResumeFunction/iac/output.tf new file mode 100644 index 0000000..9e15297 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/iac/output.tf @@ -0,0 +1,3 @@ +output "lambda_function_arn" { + value = module.lambda_function_container_image.lambda_function_arn +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeFunction/iac/variables.tf b/lambdas/ResumeProcessor/ResumeFunction/iac/variables.tf new file mode 100644 index 0000000..9f4dfdf --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/iac/variables.tf @@ -0,0 +1,19 @@ +variable "lambda_function_name" { + type = string + nullable = false +} + +variable "lambda_repository_name" { + type = string + nullable = false +} + +variable "prompt_default" { + type = string + nullable = false +} + +variable "aws_region" { + type = string + default = "us-east-1" +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeFunction/pom.xml b/lambdas/ResumeProcessor/ResumeFunction/pom.xml new file mode 100644 index 0000000..7a32e3b --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/pom.xml @@ -0,0 +1,67 @@ + + 4.0.0 + com.levio.aws-demo + resume + 1.0 + jar + Resume + + 17 + 17 + + + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-lambda-java-events + 3.11.0 + + + software.amazon.awssdk + bedrockruntime + 2.21.17 + + + software.amazon.awssdk + apache-client + 2.20.52 + + + org.json + json + 20231013 + + + junit + junit + 4.13.2 + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + + + package + + shade + + + + + + + diff --git a/lambdas/ResumeProcessor/ResumeFunction/src/main/java/com/levio/awsdemo/resume/App.java b/lambdas/ResumeProcessor/ResumeFunction/src/main/java/com/levio/awsdemo/resume/App.java new file mode 100644 index 0000000..0e7a8d7 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/src/main/java/com/levio/awsdemo/resume/App.java @@ -0,0 +1,28 @@ +package com.levio.awsdemo.resume; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.levio.awsdemo.resume.service.BedrockService; + +import java.util.Map; + +public class App implements RequestHandler, String> { + + private final BedrockService bedrockService; + + public App() { + this.bedrockService = new BedrockService(); + } + + public App(BedrockService bedrockService) { + this.bedrockService = bedrockService; + } + + public String handleRequest(final Map input, final Context context) { + String prompt = input.get("prompt"); + String text = input.get("text"); + + return bedrockService.getClaudeResponse(prompt, text); + } + +} diff --git a/lambdas/ResumeProcessor/ResumeFunction/src/main/java/com/levio/awsdemo/resume/service/BedrockService.java b/lambdas/ResumeProcessor/ResumeFunction/src/main/java/com/levio/awsdemo/resume/service/BedrockService.java new file mode 100644 index 0000000..37ab423 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/src/main/java/com/levio/awsdemo/resume/service/BedrockService.java @@ -0,0 +1,56 @@ +package com.levio.awsdemo.resume.service; + +import org.json.JSONArray; +import org.json.JSONObject; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.bedrockruntime.BedrockRuntimeClient; +import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelRequest; +import software.amazon.awssdk.services.bedrockruntime.model.InvokeModelResponse; + +public class BedrockService { + + private final BedrockRuntimeClient client = BedrockRuntimeClient.builder() + .region(Region.US_EAST_1) + .credentialsProvider(DefaultCredentialsProvider.create()) + .build(); + + private static final String PROMPT = System.getenv("PROMPT"); + + public String getClaudeResponse(String prompt, String text) { + JSONObject content = new JSONObject() + .put("type", "text") + .put("text", text); + + JSONObject message = new JSONObject() + .put("role", "user") + .put("content", new JSONArray().put(content)); + + JSONObject payload = new JSONObject() + .put("anthropic_version", "bedrock-2023-05-31") + .put("max_tokens", 4096) + .put("system", prompt != null ? prompt : PROMPT) + .put("messages", new JSONArray().put(message)); + + System.out.println("Payload: " + payload); + + InvokeModelRequest request = InvokeModelRequest.builder() + .body(SdkBytes.fromUtf8String(payload.toString())) + .modelId("anthropic.claude-3-sonnet-20240229-v1:0") + .contentType("application/json") + .accept("application/json") + .build(); + + System.out.println("Request: " + request); + + InvokeModelResponse response = client.invokeModel(request); + + JSONObject responseBody = new JSONObject(response.body().asUtf8String()); + + System.out.println("Response: " + response.body().asUtf8String()); + + return responseBody + .getJSONArray("content").getJSONObject(0).getString("text"); + } +} diff --git a/lambdas/ResumeProcessor/ResumeFunction/src/test/java/com/levio/awsdemo/resume/AppTest.java b/lambdas/ResumeProcessor/ResumeFunction/src/test/java/com/levio/awsdemo/resume/AppTest.java new file mode 100644 index 0000000..229bcbb --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/src/test/java/com/levio/awsdemo/resume/AppTest.java @@ -0,0 +1,19 @@ +package com.levio.awsdemo.resume; + +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +public class AppTest { + @Test + public void successfulResponse() { + String key = "resume/transcription/f9n7mfg2q03r35r05ro3ih8pa2og75nndj8k6v81.json"; + int lastDotIndex = key.lastIndexOf('.'); + int lastSlashIndex = key.lastIndexOf('/', lastDotIndex - 1); + String result = key.substring(lastSlashIndex + 1, lastDotIndex); + + System.out.println(result); + } +} diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/classes/com/levio/awsdemo/resume/App.class b/lambdas/ResumeProcessor/ResumeFunction/target/classes/com/levio/awsdemo/resume/App.class new file mode 100644 index 0000000..4943cf4 Binary files /dev/null and b/lambdas/ResumeProcessor/ResumeFunction/target/classes/com/levio/awsdemo/resume/App.class differ diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/classes/com/levio/awsdemo/resume/service/BedrockService.class b/lambdas/ResumeProcessor/ResumeFunction/target/classes/com/levio/awsdemo/resume/service/BedrockService.class new file mode 100644 index 0000000..2418fb8 Binary files /dev/null and b/lambdas/ResumeProcessor/ResumeFunction/target/classes/com/levio/awsdemo/resume/service/BedrockService.class differ diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/maven-archiver/pom.properties b/lambdas/ResumeProcessor/ResumeFunction/target/maven-archiver/pom.properties new file mode 100644 index 0000000..24ea413 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/target/maven-archiver/pom.properties @@ -0,0 +1,3 @@ +artifactId=resume +groupId=com.levio.aws-demo +version=1.0 diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst new file mode 100644 index 0000000..de552b7 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -0,0 +1,2 @@ +com\levio\awsdemo\resume\App.class +com\levio\awsdemo\resume\service\BedrockService.class diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst new file mode 100644 index 0000000..9cdd6a3 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -0,0 +1,2 @@ +C:\Users\b_bri\IdeaProjects\levio-aws-demo-form-backend\ResumeProcessor\ResumeFunction\src\main\java\com\levio\awsdemo\resume\service\BedrockService.java +C:\Users\b_bri\IdeaProjects\levio-aws-demo-form-backend\ResumeProcessor\ResumeFunction\src\main\java\com\levio\awsdemo\resume\App.java diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst new file mode 100644 index 0000000..e4176d2 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst @@ -0,0 +1 @@ +com\levio\awsdemo\resume\AppTest.class diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst new file mode 100644 index 0000000..37e330b --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst @@ -0,0 +1 @@ +C:\Users\b_bri\IdeaProjects\levio-aws-demo-form-backend\ResumeProcessor\ResumeFunction\src\test\java\com\levio\awsdemo\resume\AppTest.java diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/original-resume-1.0.jar b/lambdas/ResumeProcessor/ResumeFunction/target/original-resume-1.0.jar new file mode 100644 index 0000000..867021d Binary files /dev/null and b/lambdas/ResumeProcessor/ResumeFunction/target/original-resume-1.0.jar differ diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/resume-1.0.jar b/lambdas/ResumeProcessor/ResumeFunction/target/resume-1.0.jar new file mode 100644 index 0000000..0e7e4c9 Binary files /dev/null and b/lambdas/ResumeProcessor/ResumeFunction/target/resume-1.0.jar differ diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/surefire-reports/TEST-com.levio.awsdemo.resume.AppTest.xml b/lambdas/ResumeProcessor/ResumeFunction/target/surefire-reports/TEST-com.levio.awsdemo.resume.AppTest.xml new file mode 100644 index 0000000..eef1331 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/target/surefire-reports/TEST-com.levio.awsdemo.resume.AppTest.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/surefire-reports/com.levio.awsdemo.resume.AppTest.txt b/lambdas/ResumeProcessor/ResumeFunction/target/surefire-reports/com.levio.awsdemo.resume.AppTest.txt new file mode 100644 index 0000000..74dbdca --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeFunction/target/surefire-reports/com.levio.awsdemo.resume.AppTest.txt @@ -0,0 +1,4 @@ +------------------------------------------------------------------------------- +Test set: com.levio.awsdemo.resume.AppTest +------------------------------------------------------------------------------- +Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.175 s -- in com.levio.awsdemo.resume.AppTest diff --git a/lambdas/ResumeProcessor/ResumeFunction/target/test-classes/com/levio/awsdemo/resume/AppTest.class b/lambdas/ResumeProcessor/ResumeFunction/target/test-classes/com/levio/awsdemo/resume/AppTest.class new file mode 100644 index 0000000..f6c842e Binary files /dev/null and b/lambdas/ResumeProcessor/ResumeFunction/target/test-classes/com/levio/awsdemo/resume/AppTest.class differ diff --git a/lambdas/ResumeProcessor/ResumeProcessor.iml b/lambdas/ResumeProcessor/ResumeProcessor.iml new file mode 100644 index 0000000..16df530 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeProcessor.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/Dockerfile b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/Dockerfile new file mode 100644 index 0000000..45d724c --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/Dockerfile @@ -0,0 +1,15 @@ +FROM public.ecr.aws/sam/build-java17:latest as build-image + +WORKDIR "/task" +COPY src/ src/ +COPY pom.xml ./ + +RUN mvn -q clean install +RUN mvn dependency:copy-dependencies -DincludeScope=compile + +FROM public.ecr.aws/lambda/java:17 + +COPY --from=build-image /task/target/classes /var/task/ +COPY --from=build-image /task/target/dependency /var/task/lib +# Command can be overwritten by providing a different command in the template directly. +CMD ["com.levio.awsdemo.resumerequestpreprocessor.App::handleRequest"] \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/events/event.json b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/events/event.json new file mode 100644 index 0000000..b0b9d55 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/events/event.json @@ -0,0 +1,38 @@ +{ + "Records": [ + { + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": { + "principalId": "EXAMPLE" + }, + "requestParameters": { + "sourceIPAddress": "127.0.0.1" + }, + "responseElements": { + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": { + "name": "levio-demo-fev-esta-ses-bucket-dev", + "ownerIdentity": { + "principalId": "EXAMPLE" + }, + "arn": "arn:aws:s3:::example-bucket" + }, + "object": { + "key": "resume/transcription/br8ui99ml03pvltjer3f28d2vkeaserten4lmp81.json", + "size": 1024, + "eTag": "br8ui99ml03pvltjer3f28d2vkeaserten4lmp81", + "sequencer": "0A1B2C3D4E5F678901" + } + } + } + ] +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/lambda.tf b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/lambda.tf new file mode 100644 index 0000000..a659054 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/lambda.tf @@ -0,0 +1,83 @@ +data "aws_ecr_image" "lambda_image" { + repository_name = var.lambda_repository_name + most_recent = true +} + +data "aws_caller_identity" "current" {} + +module "lambda_function_container_image" { + timeout = 60 + source = "terraform-aws-modules/lambda/aws" + function_name = var.lambda_function_name + create_package = false + image_uri = data.aws_ecr_image.lambda_image.image_uri + package_type = "Image" + memory_size = 1024 + role_name = "${var.lambda_function_name}-role" + attach_policy_statements = true + + environment_variables = { + QUEUE_URL = var.queue_url + } + + policy_statements = { + log_group = { + effect = "Allow" + actions = [ + "logs:CreateLogGroup" + ] + resources = [ + "arn:aws:logs:*:*:*" + ] + } + + log_write = { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [ + "arn:aws:logs:*:*:log-group:/aws/${var.lambda_function_name}/*:*" + ] + } + + s3 = { + effect = "Allow" + actions = [ + "s3:Get*", + "s3:List*", + "s3:Describe*", + "s3:PutObject", + "s3-object-lambda:Get*", + "s3-object-lambda:List*", + "s3-object-lambda:WriteGetObjectResponse" + ] + resources = [ + var.ses_bucket_arn, + "${var.ses_bucket_arn}/*" + ] + } + + request_sqs = { + effect = "Allow" + actions = [ + "sqs:SendMessage", + ] + resources = [ + var.request_queue_arn + ] + } + + } + + create_current_version_allowed_triggers = false + + allowed_triggers = { + s3 = { + principal = "s3.amazonaws.com" + source_arn = var.ses_bucket_arn + } + } + +} diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/output.tf b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/output.tf new file mode 100644 index 0000000..9e15297 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/output.tf @@ -0,0 +1,3 @@ +output "lambda_function_arn" { + value = module.lambda_function_container_image.lambda_function_arn +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/variables.tf b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/variables.tf new file mode 100644 index 0000000..1a1ca8d --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac/variables.tf @@ -0,0 +1,29 @@ +variable "lambda_function_name" { + type = string + nullable = false +} + +variable "lambda_repository_name" { + type = string + nullable = false +} + +variable "ses_bucket_arn" { + type = string + nullable = false +} + +variable "queue_url" { + type = string + nullable = false +} + +variable "request_queue_arn" { + type = string + nullable = false +} + +variable "aws_region" { + type = string + default = "us-east-1" +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/pom.xml b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/pom.xml new file mode 100644 index 0000000..8b48bfe --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/pom.xml @@ -0,0 +1,77 @@ + + 4.0.0 + com.levio.aws-demo + resume-request-preprocessor + 1.0 + jar + Resume request preprocessor + + 17 + 17 + + + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-lambda-java-events + 3.11.0 + + + software.amazon.awssdk + s3 + 2.20.52 + + + software.amazon.awssdk + sqs + 2.20.52 + + + software.amazon.awssdk + apache-client + 2.20.52 + + + com.fasterxml.jackson.core + jackson-databind + 2.15.3 + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + 2.15.3 + + + junit + junit + 4.13.2 + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + + + package + + shade + + + + + + + diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestpreprocessor/App.java b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestpreprocessor/App.java new file mode 100644 index 0000000..0c3fbb5 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestpreprocessor/App.java @@ -0,0 +1,44 @@ +package com.levio.awsdemo.resumerequestpreprocessor; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.models.s3.S3EventNotification; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.joda.JodaModule; +import com.levio.awsdemo.resumerequestpreprocessor.service.SqsProducerService; + +public class App implements RequestHandler { + private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JodaModule()); + + private final SqsProducerService sqsProducerService; + + public App() { + this.sqsProducerService = new SqsProducerService(); + } + + public App(SqsProducerService sqsProducerService) { + this.sqsProducerService = sqsProducerService; + } + + public Void handleRequest(final S3EventNotification input, final Context context) { + try { + String json = objectMapper.writeValueAsString(input); + System.out.println(json); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + input.getRecords().forEach(s3EventNotificationRecord -> + sqsProducerService.send(extractEmailId(s3EventNotificationRecord.getS3().getObject().getKey())) + ); + + return null; + } + + private String extractEmailId(String key) { + int lastDotIndex = key.lastIndexOf('.'); + int lastSlashIndex = key.lastIndexOf('/', lastDotIndex - 1); + return key.substring(lastSlashIndex + 1, lastDotIndex); + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestpreprocessor/service/SqsProducerService.java b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestpreprocessor/service/SqsProducerService.java new file mode 100644 index 0000000..5540571 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestpreprocessor/service/SqsProducerService.java @@ -0,0 +1,23 @@ +package com.levio.awsdemo.resumerequestpreprocessor.service; + +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.SendMessageRequest; + +public class SqsProducerService { + private static final String QUEUE_URL = System.getenv("QUEUE_URL"); + private final SqsClient sqs = SqsClient.builder() + .region(Region.US_EAST_1) + .build(); + + public void send(String emailId) { + SendMessageRequest sendMessageRequest = SendMessageRequest.builder() + .queueUrl(QUEUE_URL) + .messageBody(emailId) + .messageGroupId(emailId) + .messageDeduplicationId(emailId) + .build(); + + sqs.sendMessage(sendMessageRequest); + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/test/java/com/levio/awsdemo/resumerequestpreprocessor/AppTest.java b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/test/java/com/levio/awsdemo/resumerequestpreprocessor/AppTest.java new file mode 100644 index 0000000..92ca7b7 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/src/test/java/com/levio/awsdemo/resumerequestpreprocessor/AppTest.java @@ -0,0 +1,19 @@ +package com.levio.awsdemo.resumerequestpreprocessor; + +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +public class AppTest { + @Test + public void successfulResponse() { + String key = "resume/transcription/f9n7mfg2q03r35r05ro3ih8pa2og75nndj8k6v81.json"; + int lastDotIndex = key.lastIndexOf('.'); + int lastSlashIndex = key.lastIndexOf('/', lastDotIndex - 1); + String result = key.substring(lastSlashIndex + 1, lastDotIndex); + + System.out.println(result); + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/Dockerfile b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/Dockerfile new file mode 100644 index 0000000..80ee87c --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/Dockerfile @@ -0,0 +1,15 @@ +FROM public.ecr.aws/sam/build-java17:latest as build-image + +WORKDIR "/task" +COPY src/ src/ +COPY pom.xml ./ + +RUN mvn -q clean install +RUN mvn dependency:copy-dependencies -DincludeScope=compile + +FROM public.ecr.aws/lambda/java:17 + +COPY --from=build-image /task/target/classes /var/task/ +COPY --from=build-image /task/target/dependency /var/task/lib +# Command can be overwritten by providing a different command in the template directly. +CMD ["com.levio.awsdemo.resumerequestprocessor.App::handleRequest"] \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/dependency-reduced-pom.xml b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/dependency-reduced-pom.xml new file mode 100644 index 0000000..b2979b7 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/dependency-reduced-pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + com.levio.aws-demo + resume-request-processor + Resume request processor + 1.0 + + + + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + + + + + + junit + junit + 4.13.2 + test + + + hamcrest-core + org.hamcrest + + + + + + 17 + 17 + + diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/events/event.json b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/events/event.json new file mode 100644 index 0000000..8505223 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/events/event.json @@ -0,0 +1,27 @@ +{ + "Records": [ + { + "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", + "receiptHandle": "MessageReceiptHandle", + "body": "02dq0817bnrmlk9crbam5luipgmpu2ns2fn7ogg1", + "attributes": { + "sender": "bruno.abreu@levio.ca", + "subject": "Levio HR" + }, + "messageAttributes": { + "sender": { + "stringValue": "bruno.abreu@levio.ca", + "dataType": "String" + }, + "subject": { + "stringValue": "Levio HR", + "dataType": "String" + } + }, + "md5OfBody": "{{{md5_of_body}}}", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", + "awsRegion": "us-east-1" + } + ] +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/lambda.tf b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/lambda.tf new file mode 100644 index 0000000..84d38ef --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/lambda.tf @@ -0,0 +1,113 @@ +data "aws_ecr_image" "lambda_image" { + repository_name = var.lambda_repository_name + most_recent = true +} + +data "aws_caller_identity" "current" {} + +module "lambda_function_container_image" { + timeout = 60 + source = "terraform-aws-modules/lambda/aws" + function_name = var.lambda_function_name + create_package = false + image_uri = data.aws_ecr_image.lambda_image.image_uri + package_type = "Image" + memory_size = 1024 + role_name = "${var.lambda_function_name}-role" + attach_policy_statements = true + + environment_variables = { + BUCKET_NAME = var.ses_bucket_name + FUNCTION_NAME = var.resume_function_name + QUEUE_URL = var.queue_url + DIALOGUE_PROMPT = var.dialogue_prompt + RESUME_PROMPT = var.resume_prompt + } + + policy_statements = { + log_group = { + effect = "Allow" + actions = [ + "logs:CreateLogGroup" + ] + resources = [ + "arn:aws:logs:*:*:*" + ] + } + + log_write = { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [ + "arn:aws:logs:*:*:log-group:/aws/${var.lambda_function_name}/*:*" + ] + } + + s3 = { + effect = "Allow" + actions = [ + "s3:Get*", + "s3:List*", + "s3:Describe*", + "s3:PutObject", + "s3-object-lambda:Get*", + "s3-object-lambda:List*", + "s3-object-lambda:WriteGetObjectResponse" + ] + resources = [ + var.ses_bucket_arn, + "${var.ses_bucket_arn}/*" + ] + } + + request_sqs = { + effect = "Allow" + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ] + resources = [ + module.fifo_sqs.queue_arn + ] + } + + response_sqs = { + effect = "Allow" + actions = [ + "sqs:SendMessage", + ] + resources = [ + var.response_queue_arn + ] + } + + lambda = { + effect = "Allow" + actions = [ + "lambda:InvokeFunction", + ] + resources = [ + var.resume_function_arn + ] + } + } + + create_current_version_allowed_triggers = false + + allowed_triggers = { + sqs = { + principal = "sqs.amazonaws.com" + source_arn = module.fifo_sqs.queue_arn + } + } + + event_source_mapping = { + sqs = { + event_source_arn = module.fifo_sqs.queue_arn + } + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/output.tf b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/output.tf new file mode 100644 index 0000000..ec4f86a --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/output.tf @@ -0,0 +1,11 @@ +output "lambda_function_arn" { + value = module.lambda_function_container_image.lambda_function_arn +} + +output "queue_url" { + value = module.fifo_sqs.queue_url +} + +output "queue_arn" { + value = module.fifo_sqs.queue_arn +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/sqs.tf b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/sqs.tf new file mode 100644 index 0000000..403fabb --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/sqs.tf @@ -0,0 +1,6 @@ +module "fifo_sqs" { + source = "terraform-aws-modules/sqs/aws" + name = var.sqs_name + fifo_queue = true + visibility_timeout_seconds = 60 +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/variables.tf b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/variables.tf new file mode 100644 index 0000000..f388dde --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac/variables.tf @@ -0,0 +1,59 @@ +variable "lambda_function_name" { + type = string + nullable = false +} + +variable "lambda_repository_name" { + type = string + nullable = false +} + +variable "ses_bucket_name" { + type = string + nullable = false +} + +variable "ses_bucket_arn" { + type = string + nullable = false +} + +variable "resume_function_name" { + type = string + nullable = false +} + +variable "resume_function_arn" { + type = string + nullable = false +} + +variable "queue_url" { + type = string + nullable = false +} + +variable "dialogue_prompt" { + type = string + nullable = false +} + +variable "resume_prompt" { + type = string + nullable = false +} + +variable "response_queue_arn" { + type = string + nullable = false +} + +variable "sqs_name" { + type = string + nullable = false +} + +variable "aws_region" { + type = string + default = "us-east-1" +} \ No newline at end of file diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/pom.xml b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/pom.xml new file mode 100644 index 0000000..21c2a77 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/pom.xml @@ -0,0 +1,92 @@ + + 4.0.0 + com.levio.aws-demo + resume-request-processor + 1.0 + jar + Resume request processor + + 17 + 17 + + + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-lambda-java-events + 3.11.0 + + + software.amazon.awssdk + s3 + 2.20.52 + + + software.amazon.awssdk + sqs + 2.20.52 + + + software.amazon.awssdk + lambda + 2.20.52 + + + software.amazon.awssdk + apache-client + 2.20.52 + + + com.sun.mail + jakarta.mail + 2.0.1 + + + jakarta.mail + jakarta.mail-api + 2.1.2 + + + org.projectlombok + lombok + 1.18.28 + + + org.json + json + 20210307 + + + junit + junit + 4.13.2 + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + + + package + + shade + + + + + + + diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/App.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/App.java new file mode 100644 index 0000000..88a02bd --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/App.java @@ -0,0 +1,113 @@ +package com.levio.awsdemo.resumerequestprocessor; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import com.levio.awsdemo.resumerequestprocessor.service.ClaudeService; +import com.levio.awsdemo.resumerequestprocessor.service.LambdaService; +import com.levio.awsdemo.resumerequestprocessor.service.MailService; +import com.levio.awsdemo.resumerequestprocessor.service.S3Service; +import com.levio.awsdemo.resumerequestprocessor.service.SqsProducerService; +import com.levio.awsdemo.resumerequestprocessor.util.EmailUtils; +import jakarta.mail.MessagingException; +import jakarta.mail.Part; +import jakarta.mail.internet.InternetAddress; +import jakarta.mail.internet.MimeMessage; +import org.json.JSONObject; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class App implements RequestHandler { + private final S3Service s3Service; + + private final ClaudeService claudeService; + + private final SqsProducerService sqsProducerService; + + private final MailService mailService; + + public App() { + this.s3Service = new S3Service(); + this.claudeService = new ClaudeService(new LambdaService()); + this.sqsProducerService = new SqsProducerService(); + this.mailService = new MailService(); + } + + public App(S3Service s3Service, ClaudeService claudeService, SqsProducerService sqsProducerService, MailService mailService) { + this.s3Service = s3Service; + this.claudeService = claudeService; + this.sqsProducerService = sqsProducerService; + this.mailService = mailService; + } + + public Void handleRequest(final SQSEvent input, final Context context) { + input.getRecords().forEach(record -> { + System.out.println("Record: " + record); + + String keyId = record.getBody(); + + String email = s3Service.getFile("resume/email/" + keyId); + try { + MimeMessage message = mailService.getMimeMessage(new ByteArrayInputStream(email.getBytes(StandardCharsets.UTF_8))); + String emailBody = "Resume response"; + String sender = ((InternetAddress) message.getFrom()[0]).getAddress(); + String subject = message.getSubject(); + + List attachments = EmailUtils.extractAttachments(message); + String attachmentFilename = attachments.get(0).getFileName(); + + String transcription = s3Service.getFile("resume/transcription/" + keyId + ".json"); + + String transcript = new JSONObject(transcription) + .getJSONObject("results") + .getJSONArray("transcripts") + .getJSONObject(0) + .getString("transcript"); + + String dialogue = claudeService.getDialogue(transcript); + String filename = extractFileName(attachmentFilename) + "-" + keyId; + String dialogueTxtUri = s3Service.saveFile("resume/dialogue/" + filename + ".txt", dialogue.getBytes()); + String resume = claudeService.getResume(dialogue); + String resumeTxtUri = s3Service.saveFile("resume/" + filename + ".txt", resume.getBytes()); + + sqsProducerService.send(emailBody, getMessageAttributes(sender, subject, dialogueTxtUri, resumeTxtUri), keyId); + } catch (MessagingException | IOException e) { + throw new RuntimeException(e); + } + }); + + return null; + } + + private String extractFileName(String fileName) { + return fileName.substring(0, fileName.lastIndexOf(".")); + } + + private static Map getMessageAttributes(String sender, String subject, String... attachmentsUri) { + Map messageAttributes = new HashMap<>(); + + SQSEvent.MessageAttribute senderAttribute = new SQSEvent.MessageAttribute(); + senderAttribute.setStringValue(sender); + senderAttribute.setDataType("String"); + messageAttributes.put("sender", senderAttribute); + + SQSEvent.MessageAttribute subjectAttribute = new SQSEvent.MessageAttribute(); + subjectAttribute.setStringValue(subject); + subjectAttribute.setDataType("String"); + messageAttributes.put("subject", subjectAttribute); + + for (int i = 0; i < attachmentsUri.length; i++) { + SQSEvent.MessageAttribute attachmentAttribute = new SQSEvent.MessageAttribute(); + attachmentAttribute.setStringValue(attachmentsUri[i]); + attachmentAttribute.setDataType("String"); + messageAttributes.put("attachment" + i, attachmentAttribute); + } + + return messageAttributes; + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/ClaudeService.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/ClaudeService.java new file mode 100644 index 0000000..0e88855 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/ClaudeService.java @@ -0,0 +1,29 @@ +package com.levio.awsdemo.resumerequestprocessor.service; + +import lombok.RequiredArgsConstructor; +import org.json.JSONObject; + +@RequiredArgsConstructor +public class ClaudeService { + private static final String FUNCTION_NAME = System.getenv("FUNCTION_NAME"); + private static final String DIALOGUE_PROMPT = System.getenv("DIALOGUE_PROMPT"); + private static final String RESUME_PROMPT = System.getenv("RESUME_PROMPT"); + + private final LambdaService lambdaService; + + public String getDialogue(String transcription) { + JSONObject payload = new JSONObject() + .put("prompt", DIALOGUE_PROMPT) + .put("text", transcription); + + return lambdaService.invoke(FUNCTION_NAME, payload.toString()); + } + + public String getResume(String dialogue) { + JSONObject payload = new JSONObject() + .put("prompt", RESUME_PROMPT) + .put("text", dialogue); + + return lambdaService.invoke(FUNCTION_NAME, payload.toString()); + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/LambdaService.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/LambdaService.java new file mode 100644 index 0000000..1eae80f --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/LambdaService.java @@ -0,0 +1,24 @@ +package com.levio.awsdemo.resumerequestprocessor.service; + +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.lambda.LambdaClient; +import software.amazon.awssdk.services.lambda.model.InvokeRequest; +import software.amazon.awssdk.services.lambda.model.InvokeResponse; + +public class LambdaService { + private final LambdaClient lambda = LambdaClient.builder() + .region(Region.US_EAST_1) + .build(); + + public String invoke(String functionName, String payload) { + InvokeRequest invokeRequest = InvokeRequest.builder() + .functionName(functionName) + .payload(SdkBytes.fromUtf8String(payload)) + .build(); + + InvokeResponse invokeResponse = lambda.invoke(invokeRequest); + + return invokeResponse.payload().asUtf8String(); + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/MailService.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/MailService.java new file mode 100644 index 0000000..2bd20dd --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/MailService.java @@ -0,0 +1,18 @@ +package com.levio.awsdemo.resumerequestprocessor.service; + +import jakarta.mail.MessagingException; +import jakarta.mail.Session; +import jakarta.mail.internet.MimeMessage; + +import java.io.InputStream; +import java.util.Properties; + +public class MailService { + + public MimeMessage getMimeMessage(InputStream inputStream) throws MessagingException { + Properties props = new Properties(); + Session session = Session.getDefaultInstance(props, null); + + return new MimeMessage(session, inputStream); + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/S3Service.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/S3Service.java new file mode 100644 index 0000000..c829238 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/S3Service.java @@ -0,0 +1,47 @@ +package com.levio.awsdemo.resumerequestprocessor.service; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; + +import java.io.ByteArrayInputStream; + +public class S3Service { + + private static final String BUCKET_NAME = System.getenv("BUCKET_NAME"); + + private final S3Client s3 = S3Client.builder() + .region(Region.US_EAST_1) + .build(); + + public String getFile(String key) { + GetObjectRequest objectRequest = GetObjectRequest + .builder() + .key(key) + .bucket(BUCKET_NAME) + .build(); + + ResponseBytes objectBytes = s3.getObjectAsBytes(objectRequest); + return new String(objectBytes.asByteArray()); + } + + public String saveFile(String fileKey, byte[] fileContent) { + PutObjectResponse objectResponse = s3.putObject( + PutObjectRequest.builder() + .bucket(BUCKET_NAME) + .key(fileKey) + .build(), + RequestBody.fromBytes(fileContent)); + if (objectResponse.sdkHttpResponse().isSuccessful()) { + System.out.println("File " + fileKey + " created"); + return "s3://" + BUCKET_NAME + "/" + fileKey; + } + return null; + } + +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/SqsProducerService.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/SqsProducerService.java new file mode 100644 index 0000000..32faf90 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/service/SqsProducerService.java @@ -0,0 +1,38 @@ +package com.levio.awsdemo.resumerequestprocessor.service; + +import com.amazonaws.services.lambda.runtime.events.SQSEvent; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.MessageAttributeValue; +import software.amazon.awssdk.services.sqs.model.SendMessageRequest; + +import java.util.HashMap; +import java.util.Map; + +public class SqsProducerService { + private static final String QUEUE_URL = System.getenv("QUEUE_URL"); + private final SqsClient sqs = SqsClient.builder() + .region(Region.US_EAST_1) + .build(); + + public void send(String message, Map messageAttributes, String messageId) { + Map messageAttributeValues = new HashMap<>(); + messageAttributes.forEach((key, value) -> { + MessageAttributeValue mav = MessageAttributeValue.builder() + .stringValue(value.getStringValue()) + .dataType(value.getDataType()) + .build(); + messageAttributeValues.put(key, mav); + }); + + SendMessageRequest sendMessageRequest = SendMessageRequest.builder() + .queueUrl(QUEUE_URL) + .messageBody(message) + .messageAttributes(messageAttributeValues) + .messageGroupId(messageAttributes.get("sender").getStringValue()) + .messageDeduplicationId(messageId) + .build(); + + sqs.sendMessage(sendMessageRequest); + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/util/EmailUtils.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/util/EmailUtils.java new file mode 100644 index 0000000..6f948f2 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/main/java/com/levio/awsdemo/resumerequestprocessor/util/EmailUtils.java @@ -0,0 +1,28 @@ +package com.levio.awsdemo.resumerequestprocessor.util; + +import jakarta.mail.MessagingException; +import jakarta.mail.Multipart; +import jakarta.mail.Part; +import jakarta.mail.internet.MimeMessage; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class EmailUtils { + public static List extractAttachments(MimeMessage message) throws MessagingException, IOException { + Object content = message.getContent(); + + List attachments = new ArrayList<>(); + + if (content instanceof Multipart multipart) { + for (int i = 0; i < multipart.getCount(); i++) { + Part part = multipart.getBodyPart(i); + if (Part.ATTACHMENT.equalsIgnoreCase(part.getDisposition())) { + attachments.add(part); + } + } + } + return attachments; + } +} diff --git a/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/test/java/com/levio/awsdemo/resumerequestprocessor/AppTest.java b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/test/java/com/levio/awsdemo/resumerequestprocessor/AppTest.java new file mode 100644 index 0000000..92b77d8 --- /dev/null +++ b/lambdas/ResumeProcessor/ResumeRequestProcessorFunction/src/test/java/com/levio/awsdemo/resumerequestprocessor/AppTest.java @@ -0,0 +1,14 @@ +//package com.levio.awsdemo.resumerequestprocessor; +// +//import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +//import static org.junit.Assert.assertEquals; +//import static org.junit.Assert.assertNotNull; +//import static org.junit.Assert.assertTrue; +//import org.junit.Test; +// +//public class AppTest { +// @Test +// public void successfulResponse() { +// +// } +//} diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/Dockerfile b/lambdas/TranscriptionProcessor/TranscriptionFunction/Dockerfile new file mode 100644 index 0000000..016ac88 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/Dockerfile @@ -0,0 +1,15 @@ +FROM public.ecr.aws/sam/build-java17:latest as build-image + +WORKDIR "/task" +COPY src/ src/ +COPY pom.xml ./ + +RUN mvn -q clean install +RUN mvn dependency:copy-dependencies -DincludeScope=compile + +FROM public.ecr.aws/lambda/java:17 + +COPY --from=build-image /task/target/classes /var/task/ +COPY --from=build-image /task/target/dependency /var/task/lib +# Command can be overwritten by providing a different command in the template directly. +CMD ["com.levio.awsdemo.transcription.App::handleRequest"] \ No newline at end of file diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/dependency-reduced-pom.xml b/lambdas/TranscriptionProcessor/TranscriptionFunction/dependency-reduced-pom.xml new file mode 100644 index 0000000..43344a1 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/dependency-reduced-pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + com.levio.aws-demo + transcription + Transcription processor + 1.0 + + + + maven-shade-plugin + 3.2.4 + + + package + + shade + + + + + + + + + + junit + junit + 4.13.2 + test + + + hamcrest-core + org.hamcrest + + + + + + 17 + 17 + + diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/events/event.json b/lambdas/TranscriptionProcessor/TranscriptionFunction/events/event.json new file mode 100644 index 0000000..45a15e1 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/events/event.json @@ -0,0 +1,38 @@ +{ + "Records": [ + { + "eventVersion": "2.0", + "eventSource": "aws:s3", + "awsRegion": "us-east-1", + "eventTime": "1970-01-01T00:00:00.000Z", + "eventName": "ObjectCreated:Put", + "userIdentity": { + "principalId": "EXAMPLE" + }, + "requestParameters": { + "sourceIPAddress": "127.0.0.1" + }, + "responseElements": { + "x-amz-request-id": "EXAMPLE123456789", + "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "testConfigRule", + "bucket": { + "name": "levio-demo-fev-esta-ses-bucket-dev", + "ownerIdentity": { + "principalId": "EXAMPLE" + }, + "arn": "arn:aws:s3:::example-bucket" + }, + "object": { + "key": "resume/audio/exemple1.mp3", + "size": 1024, + "eTag": "0123456789abcdef0123456789abcdef", + "sequencer": "0A1B2C3D4E5F678901" + } + } + } + ] +} \ No newline at end of file diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/lambda.tf b/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/lambda.tf new file mode 100644 index 0000000..7f6f1b7 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/lambda.tf @@ -0,0 +1,103 @@ +data "aws_ecr_image" "lambda_image" { + repository_name = var.lambda_repository_name + most_recent = true +} + +data "aws_caller_identity" "current" {} + +module "lambda_function_container_image" { + timeout = 60 + source = "terraform-aws-modules/lambda/aws" + function_name = var.lambda_function_name + create_package = false + image_uri = data.aws_ecr_image.lambda_image.image_uri + package_type = "Image" + memory_size = 1024 + role_name = "${var.lambda_function_name}-role" + attach_policy_statements = true + + environment_variables = { + DATA_ACCESS_ROLE_ARN = module.lambda_function_container_image.lambda_role_arn + } + + trusted_entities = [ + { + type = "Service", + identifiers = [ + "transcribe.amazonaws.com", + "lambda.amazonaws.com" + ] + } + ] + + policy_statements = { + log_group = { + effect = "Allow" + actions = [ + "logs:CreateLogGroup" + ] + resources = [ + "arn:aws:logs:*:*:*" + ] + } + + log_write = { + effect = "Allow" + actions = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + resources = [ + "arn:aws:logs:*:*:log-group:/aws/${var.lambda_function_name}/*:*" + ] + } + + s3 = { + effect = "Allow" + actions = [ + "s3:Get*", + "s3:List*", + "s3:Describe*", + "s3:PutObject", + "s3-object-lambda:Get*", + "s3-object-lambda:List*", + "s3-object-lambda:WriteGetObjectResponse" + ] + resources = [ + var.ses_bucket_arn, + "${var.ses_bucket_arn}/*" + ] + } + + transcribe = { + effect = "Allow" + actions = [ + "transcribe:StartTranscriptionJob", + "transcribe:GetTranscriptionJob" + ] + resources = [ + "arn:aws:transcribe:${var.aws_region}:${data.aws_caller_identity.current.account_id}:transcription-job/*" + ] + } + + iam = { + effect = "Allow" + actions = [ + "iam:PassRole" + ] + resources = [ + module.lambda_function_container_image.lambda_role_arn + ] + } + } + + create_current_version_allowed_triggers = false + + allowed_triggers = { + s3 = { + principal = "s3.amazonaws.com" + source_arn = var.ses_bucket_arn + } + } + +} diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/output.tf b/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/output.tf new file mode 100644 index 0000000..9e15297 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/output.tf @@ -0,0 +1,3 @@ +output "lambda_function_arn" { + value = module.lambda_function_container_image.lambda_function_arn +} \ No newline at end of file diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/variables.tf b/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/variables.tf new file mode 100644 index 0000000..06e90d8 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/iac/variables.tf @@ -0,0 +1,24 @@ +variable "lambda_function_name" { + type = string + nullable = false +} + +variable "lambda_repository_name" { + type = string + nullable = false +} + +variable "ses_bucket_name" { + type = string + nullable = false +} + +variable "ses_bucket_arn" { + type = string + nullable = false +} + +variable "aws_region" { + type = string + default = "us-east-1" +} \ No newline at end of file diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/pom.xml b/lambdas/TranscriptionProcessor/TranscriptionFunction/pom.xml new file mode 100644 index 0000000..741dee0 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/pom.xml @@ -0,0 +1,73 @@ + + 4.0.0 + com.levio.aws-demo + transcription + 1.0 + jar + Transcription processor + + 17 + 17 + + + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-lambda-java-events + 3.11.0 + + + software.amazon.awssdk + transcribe + 2.21.20 + + + software.amazon.awssdk + apache-client + 2.21.20 + + + junit + junit + 4.13.2 + test + + + com.fasterxml.jackson.core + jackson-databind + 2.15.3 + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + 2.15.3 + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.4 + + + + + package + + shade + + + + + + + diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/src/main/java/com/levio/awsdemo/transcription/App.java b/lambdas/TranscriptionProcessor/TranscriptionFunction/src/main/java/com/levio/awsdemo/transcription/App.java new file mode 100644 index 0000000..e708424 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/src/main/java/com/levio/awsdemo/transcription/App.java @@ -0,0 +1,37 @@ +package com.levio.awsdemo.transcription; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.amazonaws.services.lambda.runtime.events.models.s3.S3EventNotification; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.joda.JodaModule; +import com.levio.awsdemo.transcription.service.TranscribeService; + +public class App implements RequestHandler { + + private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JodaModule()); + + private final TranscribeService transcribeService; + + public App(TranscribeService transcribeService) { + this.transcribeService = transcribeService; + } + + public App() { + this.transcribeService = new TranscribeService(); + } + + public Void handleRequest(final S3EventNotification input, final Context context) { + System.out.println(input); + try { + String json = objectMapper.writeValueAsString(input); + System.out.println(json); + input.getRecords().forEach(s3EventNotificationRecord -> transcribeService.transcribe(s3EventNotificationRecord.getS3())); + + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return null; + } +} diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/src/main/java/com/levio/awsdemo/transcription/service/TranscribeService.java b/lambdas/TranscriptionProcessor/TranscriptionFunction/src/main/java/com/levio/awsdemo/transcription/service/TranscribeService.java new file mode 100644 index 0000000..ec6bbdc --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/src/main/java/com/levio/awsdemo/transcription/service/TranscribeService.java @@ -0,0 +1,95 @@ +package com.levio.awsdemo.transcription.service; + +import com.amazonaws.services.lambda.runtime.events.models.s3.S3EventNotification; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.transcribe.TranscribeClient; +import software.amazon.awssdk.services.transcribe.model.GetTranscriptionJobRequest; +import software.amazon.awssdk.services.transcribe.model.GetTranscriptionJobResponse; +import software.amazon.awssdk.services.transcribe.model.JobExecutionSettings; +import software.amazon.awssdk.services.transcribe.model.LanguageCode; +import software.amazon.awssdk.services.transcribe.model.Media; +import software.amazon.awssdk.services.transcribe.model.Settings; +import software.amazon.awssdk.services.transcribe.model.StartTranscriptionJobRequest; +import software.amazon.awssdk.services.transcribe.model.StartTranscriptionJobResponse; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TranscribeService { + + private final String DATA_ACCESS_ROLE_ARN = System.getenv("DATA_ACCESS_ROLE_ARN"); + private final TranscribeClient client = TranscribeClient.builder() + .credentialsProvider(getCredentials()) + .region(Region.US_EAST_1) + .build(); + + public void transcribe(S3EventNotification.S3Entity s3) { + String filename = retrieveFileName(s3.getObject().getKey()); + System.out.println(filename); + String transcriptionJobName = filename + "-transcription-job"; + String mediaType = retrieveExtension(s3.getObject().getKey()); + String s3FileUrl = "s3://" + s3.getBucket().getName() + "/" + s3.getObject().getKey(); + System.out.println(s3FileUrl); + Media myMedia = Media.builder() + .mediaFileUri(s3FileUrl) + .build(); + + StartTranscriptionJobRequest request = StartTranscriptionJobRequest.builder() + .transcriptionJobName(transcriptionJobName) + .identifyLanguage(true) + .languageOptions(LanguageCode.EN_US, LanguageCode.FR_CA) + .mediaFormat(mediaType) + .media(myMedia) + .outputBucketName(s3.getBucket().getName()) + .outputKey("resume/transcription/" + filename + ".json") + .settings(Settings.builder() + .showSpeakerLabels(true) + .maxSpeakerLabels(3) + .build()) + .jobExecutionSettings(JobExecutionSettings.builder() + .allowDeferredExecution(true) + .dataAccessRoleArn(DATA_ACCESS_ROLE_ARN) + .build()) + .build(); + + System.out.println(request); + + StartTranscriptionJobResponse startJobResponse = client.startTranscriptionJob(request); + + System.out.println("Start job response - Transcription job"); + System.out.println(startJobResponse.transcriptionJob()); + + GetTranscriptionJobRequest getJobRequest = GetTranscriptionJobRequest.builder() + .transcriptionJobName(transcriptionJobName) + .build(); + + GetTranscriptionJobResponse getJobResponse = client.getTranscriptionJob(getJobRequest); + + System.out.println("Get job response - Transcription job"); + System.out.println(getJobResponse.transcriptionJob()); + } + + private String retrieveFileName(String key) { + Pattern pattern = Pattern.compile(".*/(.*?)\\..*"); + Matcher matcher = pattern.matcher(key); + if (matcher.find()) { + return matcher.group(1); + } + return null; + } + + private String retrieveExtension(String key) { + Pattern pattern = Pattern.compile("\\.([^.]+)$"); + Matcher matcher = pattern.matcher(key); + if (matcher.find()) { + return matcher.group(1); + } + return null; + } + + private static AwsCredentialsProvider getCredentials() { + return DefaultCredentialsProvider.create(); + } +} diff --git a/lambdas/TranscriptionProcessor/TranscriptionFunction/src/test/java/com/levio/awsdemo/transcription/AppTest.java b/lambdas/TranscriptionProcessor/TranscriptionFunction/src/test/java/com/levio/awsdemo/transcription/AppTest.java new file mode 100644 index 0000000..4a2167e --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionFunction/src/test/java/com/levio/awsdemo/transcription/AppTest.java @@ -0,0 +1,22 @@ +package com.levio.awsdemo.transcription; + +import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + +public class AppTest { + @Test + public void successfulResponse() { +// App app = new App(); +// APIGatewayProxyResponseEvent result = app.handleRequest(null, null); +// assertEquals(200, result.getStatusCode().intValue()); +// assertEquals("application/json", result.getHeaders().get("Content-Type")); +// String content = result.getBody(); +// assertNotNull(content); +// assertTrue(content.contains("\"message\"")); +// assertTrue(content.contains("\"hello world\"")); +// assertTrue(content.contains("\"location\"")); + } +} diff --git a/lambdas/TranscriptionProcessor/TranscriptionProcessor.iml b/lambdas/TranscriptionProcessor/TranscriptionProcessor.iml new file mode 100644 index 0000000..16df530 --- /dev/null +++ b/lambdas/TranscriptionProcessor/TranscriptionProcessor.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/terraform/modules.tf b/terraform/modules.tf index 2499162..9f2a4a6 100644 --- a/terraform/modules.tf +++ b/terraform/modules.tf @@ -1,17 +1,23 @@ locals { - memory_lambda_name = "levio-demo-fev-memory-dev" - dynamo_history_table_name = "levio-demo-fev-chat-history-dev" - storage_bucket_name = "levio-demo-fev-storage-dev" - queue_name = "levio-demo-fev-ingestion-queue-dev" - ingestion_lambda_name = "levio-demo-fev-ingestion-dev" - inference_lambda_name = "levio-demo-fev-inference-dev" - list_collections_lambda_name = "levio-demo-fev-list-collections-dev" - lex_router_lambda_name = "levio-demo-fev-lex-router-dev" - email_request_preprocessor_lambda_name = "levio-demo-fev-email-request-preprocessor-dev" - email_request_processor_lambda_name = "levio-demo-fev-email-request-processor-dev" - email_request_processor_queue_name = "levio-demo-fev-email-request-processor-queue-dev" - email_response_processor_lambda_name = "levio-demo-fev-email-response-processor-dev" - email_response_processor_queue_name = "levio-demo-fev-email-response-processor-queue-dev" + memory_lambda_name = "levio-demo-fev-memory-dev" + dynamo_history_table_name = "levio-demo-fev-chat-history-dev" + storage_bucket_name = "levio-demo-fev-storage-dev" + queue_name = "levio-demo-fev-ingestion-queue-dev" + ingestion_lambda_name = "levio-demo-fev-ingestion-dev" + inference_lambda_name = "levio-demo-fev-inference-dev" + list_collections_lambda_name = "levio-demo-fev-list-collections-dev" + lex_router_lambda_name = "levio-demo-fev-lex-router-dev" + email_request_preprocessor_lambda_name = "levio-demo-fev-email-request-preprocessor-dev" + email_request_processor_lambda_name = "levio-demo-fev-email-request-processor-dev" + email_request_processor_queue_name = "levio-demo-fev-email-request-processor-queue-dev" + email_response_processor_lambda_name = "levio-demo-fev-email-response-processor-dev" + email_response_processor_queue_name = "levio-demo-fev-email-response-processor-queue-dev" + attachment_saver_lambda_name = "levio-demo-fev-attachment-saver-dev" + transcription_processor_lambda_name = "levio-demo-fev-transcription-processor-dev" + resume_lambda_name = "levio-demo-fev-resume-dev" + resume_request_processor_lambda_name = "levio-demo-fev-resume-request-processor-dev" + resume_request_preprocessor_lambda_name = "levio-demo-fev-resume-request-preprocessor-dev" + resume_request_processor_queue_name = "levio-demo-fev-resume-request-processor-queue-dev" } module "ingestion" { @@ -122,10 +128,57 @@ module "email_request_preprocessor" { lambda_function_name = local.email_request_preprocessor_lambda_name lambda_repository_name = var.email_request_preprocessor_lambda_repository_name ses_bucket_name = local.bucket_name - chat_key_prefix = local.chat_key_prefix + chat_key_prefix = local.chat_key_prefix request_queue_url = module.email_request_processor.queue_url request_queue_arn = module.email_request_processor.queue_arn ses_s3_arn = module.s3_bucket.s3_bucket_arn rule_set_name = local.rule_set_name chat_rule_name = local.chat_rule_name +} + +module "attachment_saver" { + source = "../lambdas/AttachmentSaver/AttachmentSaverFunction/iac" + lambda_function_name = local.attachment_saver_lambda_name + lambda_repository_name = var.attachment_saver_lambda_repository_name + ses_bucket_name = local.bucket_name + ses_bucket_arn = module.s3_bucket.s3_bucket_arn +} + +module "transcription_processor" { + source = "../lambdas/TranscriptionProcessor/TranscriptionFunction/iac" + lambda_function_name = local.transcription_processor_lambda_name + lambda_repository_name = var.transcription_processor_lambda_repository_name + ses_bucket_name = local.bucket_name + ses_bucket_arn = module.s3_bucket.s3_bucket_arn +} + +module "resume" { + source = "../lambdas/ResumeProcessor/ResumeFunction/iac" + lambda_function_name = local.resume_lambda_name + lambda_repository_name = var.resume_lambda_repository_name + prompt_default = var.prompt_default +} + +module "resume_request_processor" { + source = "../lambdas/ResumeProcessor/ResumeRequestProcessorFunction/iac" + lambda_function_name = local.resume_request_processor_lambda_name + lambda_repository_name = var.resume_request_processor_lambda_repository_name + ses_bucket_name = local.bucket_name + ses_bucket_arn = module.s3_bucket.s3_bucket_arn + resume_function_name = local.resume_lambda_name + resume_function_arn = module.resume.lambda_function_arn + response_queue_arn = module.email_response_processor.queue_arn + queue_url = module.email_response_processor.queue_url + dialogue_prompt = var.dialogue_prompt + resume_prompt = var.resume_prompt + sqs_name = local.resume_request_processor_queue_name +} + +module "resume_request_preprocessor" { + source = "../lambdas/ResumeProcessor/ResumeRequestPreProcessorFunction/iac" + lambda_function_name = local.resume_request_preprocessor_lambda_name + lambda_repository_name = var.resume_request_preprocessor_lambda_repository_name + ses_bucket_arn = module.s3_bucket.s3_bucket_arn + request_queue_arn = module.resume_request_processor.queue_arn + queue_url = module.resume_request_processor.queue_url } \ No newline at end of file diff --git a/terraform/ses.tf b/terraform/ses.tf index 84f7a52..1877b20 100644 --- a/terraform/ses.tf +++ b/terraform/ses.tf @@ -58,4 +58,27 @@ data "aws_iam_policy_document" "allow_access_from_ses" { "${module.s3_bucket.s3_bucket_arn}/*", ] } +} + +resource "aws_s3_bucket_notification" "bucket_notification" { + bucket = local.bucket_name + + lambda_function { + lambda_function_arn = module.transcription_processor.lambda_function_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "resume/attachment/" + } + + lambda_function { + lambda_function_arn = module.attachment_saver.lambda_function_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "resume/email/" + } + + lambda_function { + lambda_function_arn = module.resume_request_preprocessor.lambda_function_arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "resume/transcription/" + } + } \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index ff60d0d..9b917ec 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -70,4 +70,44 @@ variable "sender_email" { variable "chat_rule_recipient" { type = string nullable = false +} + +variable "attachment_saver_lambda_repository_name" { + type = string + nullable = false +} + +variable "transcription_processor_lambda_repository_name" { + type = string + nullable = false +} + +variable "resume_lambda_repository_name" { + type = string + nullable = false +} + +variable "resume_request_processor_lambda_repository_name" { + type = string + nullable = false +} + +variable "resume_request_preprocessor_lambda_repository_name" { + type = string + nullable = false +} + +variable "prompt_default" { + default = "Refais ce texte sous forme de dialogues entre un intervenant et ses clients: " + type = string +} + +variable "dialogue_prompt" { + default = "Refais ce texte sous forme de dialogues entre un intervenant et ses clients: " + type = string +} + +variable "resume_prompt" { + default = "Tu es un travailleur social. Fais une analyse de ce texte. Ne résume pas trop, permets toi d'avoir du contenu pour soutenir ta réponse" + type = string } \ No newline at end of file