diff --git a/src/main/config/secor.dev.partition.properties b/src/main/config/secor.dev.partition.properties index 24af2c9f1..d198ee199 100644 --- a/src/main/config/secor.dev.partition.properties +++ b/src/main/config/secor.dev.partition.properties @@ -19,10 +19,10 @@ include=secor.dev.properties secor.kafka.group=secor_partition # Parser class that extracts s3 partitions from consumed messages. -secor.message.parser.class=com.pinterest.secor.parser.PatternDateMessageParser +secor.message.parser.class=com.pinterest.secor.parser.ThriftMessageParser # S3 path where sequence files are stored. -secor.s3.path=secor_dev +secor.s3.path=secor_dev/partition # Local path where sequence files are stored before they are uploaded to s3. secor.local.path=/tmp/secor_dev/message_logs/partition @@ -30,5 +30,5 @@ secor.local.path=/tmp/secor_dev/message_logs/partition # Port of the Ostrich server. ostrich.port=9998 -# Partition Date Output format. This is used along with PatternDateMessageParser. -secor.partition.output_dt_format=dd_MM_yyyy \ No newline at end of file +# Partition Date Output format. This is used along with PatternDateMessageParser. Defaults to 'yyyy-MM-dd' *New* +secor.partition.output_dt_format= \ No newline at end of file diff --git a/src/main/config/secor.dev.properties b/src/main/config/secor.dev.properties index a53fea7b9..8c7509d94 100644 --- a/src/main/config/secor.dev.properties +++ b/src/main/config/secor.dev.properties @@ -5,7 +5,7 @@ include=secor.common.properties ############ # Name of the s3 bucket where log files are stored. -secor.s3.bucket=lpdev-ekstep +secor.s3.bucket= ################ # END MUST SET # @@ -17,10 +17,12 @@ kafka.seed.broker.port=9092 zookeeper.quorum=localhost:2181 # Upload policies. -# 100M -secor.max.file.size.bytes=1000000000 -# 1 minute -secor.max.file.age.seconds=60 +# 10K +secor.max.file.size.bytes=10000 +# 10 seconds +secor.max.file.age.seconds=10 +# Compute the max file age for a topic partition using either the oldest or newest file. Defaults to newest +secor.max.file.age.policy= # Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz. # Available placeholders are