Skip to content

Commit

Permalink
lambda code working
Browse files Browse the repository at this point in the history
  • Loading branch information
clamorisse committed Oct 25, 2016
1 parent 4467a78 commit 207bbc3
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 0 deletions.
2 changes: 2 additions & 0 deletions function/example.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{u'Records': [{u'eventVersion': u'2.0', u'eventTime': u'2016-10-24T23:43:56.131Z', u'requestParameters': {u'sourceIPAddress': u'71.203.118.93'}, u's3': {u'configurationId': u'tf-s3-lambda-20161024145130057031816mjv', u'object': {u'eTag': u'5837a3bda61ff120afd4c352774d286f', u'sequencer': u'00580E9CBC0DE80DD2', u'key': u'data/CA4PExc1000.csv', u'size': 1850}, u'bucket': {u'arn': u'arn:aws:s3:::source_bvc_files', u'name': u'source_bvc_files', u'ownerIdentity': {u'principalId': u'A1H0DJEM19Y0K2'}}, u's3SchemaVersion': u'1.0'}, u'responseElements': {u'x-amz-id-2': u'nCOtO1sXZGDADX14cerU9TFmLzF68mbhQoicx8uGiQ8UTWD/1bUqGgnH5H5Vn6q8FNRetriQFuw=', u'x-amz-request-id': u'D1F921C4C6BBF976'}, u'awsRegion': u'us-east-1', u'eventName': u'ObjectCreated:Put', u'userIdentity': {u'principalId': u'AWS:AIDAIHCYA4J546IECO44E'}, u'eventSource': u'aws:s3'}]}

Binary file added function/lambda_analyze_csv.zip
Binary file not shown.
70 changes: 70 additions & 0 deletions function/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
from __future__ import print_function
import csv
import os
import sys
import uuid
import ntpath
import boto3
import logging


s3_client = boto3.client("s3")

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def create_dirs(path):
dir = os.path.dirname(path)
if not os.path.exists(dir):
os.makedirs(dir)

def analyze_data(input_file_path, output_file_path, file_name):
fieldnames = ["wavelength", "intensity"]
with open(input_file_path) as input_file:

data = csv.DictReader(input_file, fieldnames=fieldnames)
logger.info("PA-1: data, %s", data)

max_intensity = None
max_wavelength = None

# Find max intensity
for row in data:
intensity = float(row["intensity"])
if max_intensity == None or max_intensity < intensity:
max_intensity = intensity
max_wavelength = int(row["wavelength"])
logger.info("PA-2: max_intensity, max_wavelength %s %s", max_intensity, max_wavelength)

if max_intensity:
max_values = { "concentration": int(file_name[7:-4]), "wavelength": max_wavelength, "intensity": max_intensity }
#max_values = { "wavelength": max_wavelength, "intensity": max_intensity }
logger.info("PA-3: found max values for intensity, wavelength %s %s", max_intensity, max_wavelength)
out_field_names = max_values.keys()
with open(output_file_path, "wb") as output_file:
logger.info("PA-4: writing max values for intensity, wavelength %s", max_values)
writer = csv.DictWriter(output_file, fieldnames = out_field_names)
writer.writerow(max_values)

def handle(event, context):
for record in event['Records']:
logger.info("PY-1: Event, Context %s, %s", event, context)
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']
logger.info("PY-2: bucket_name %s", bucket)
logger.info("PY-3: key %s", key)

download_path = '/tmp/{}{}'.format(uuid.uuid4(), key)
create_dirs(download_path)
upload_path = '/tmp/analyzed-{}'.format(key)
create_dirs(upload_path)

object_name = os.path.basename(download_path)
logger.info("PY-4: downloaded location and filename: %s %s", download_path, object_name)
logger.info("PY-4: upload location: %s", upload_path)

s3_client.download_file(bucket, key, download_path)
analyze_data(download_path, upload_path, object_name)
logger.info("PY-5: uploading file to S3 (path, bucket, key) %s %s %s", upload_path, '{}analyzed'.format(bucket), key)
s3_client.upload_file(upload_path, '{}analyzed'.format(bucket), key)

0 comments on commit 207bbc3

Please sign in to comment.