Reset property file changes

src/main/config/log4j.dev.properties

@@ -4,26 +4,15 @@
 log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
 
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.Threshold=WARN
+log4j.appender.CONSOLE.Threshold=INFO
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
 log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] (%C:%L) %-5p %m%n
 
 log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
-log4j.appender.ROLLINGFILE.Threshold=INFO
-log4j.appender.ROLLINGFILE.File=/tmp/secor-me/logs/secor-${secor_group}.log
+log4j.appender.ROLLINGFILE.Threshold=DEBUG
+log4j.appender.ROLLINGFILE.File=/tmp/secor_dev/logs/secor.log
 # keep log files up to 1G
 log4j.appender.ROLLINGFILE.MaxFileSize=20MB
 log4j.appender.ROLLINGFILE.MaxBackupIndex=50
 log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [%t] (%C:%L) %-5p %m%n
-
-log4j.appender.ROLLINGFILE2=org.apache.log4j.RollingFileAppender
-log4j.appender.ROLLINGFILE2.Threshold=DEBUG
-log4j.appender.ROLLINGFILE2.File=/tmp/secor-me/logs/secor-uploader.log
-# keep log files up to 1G
-log4j.appender.ROLLINGFILE2.MaxFileSize=200MB
-log4j.appender.ROLLINGFILE2.MaxBackupIndex=5
-log4j.appender.ROLLINGFILE2.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROLLINGFILE2.layout.ConversionPattern=%d{ISO8601} [%t] (%C:%L) %-5p %m%n
-
-log4j.logger.com.pinterest.secor.uploader.Uploader=DEBUG, ROLLINGFILE2
+log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [%t] (%C:%L) %-5p %m%n

src/main/config/secor.common.properties

@@ -18,15 +18,11 @@
 ############
 
 # Regular expression matching names of consumed topics.
-<<<<<<< HEAD
-secor.kafka.topic_filter=sandbox.telemetry.unique
-=======
-secor.kafka.topic_filter=replay
->>>>>>> 629fec9a3ecb48e5b2ed4b6933d129b6c32fb40e
+secor.kafka.topic_filter=*
 
 # AWS authentication credentials.
-aws.access.key=${aws.access.key}
-aws.secret.key=${aws.secret.key}
+aws.access.key=
+aws.secret.key=
 
 ################
 # END MUST SET #
@@ -132,19 +128,21 @@ monitoring.blacklist.topics=
 statsd.hostport=
 
 # Name of field that contains timestamp for JSON, MessagePack, or Thrift message parser. (1405970352123)
-message.timestamp.name=@timestamp
+message.timestamp.name=
 
-# Name of field that contains timestamp for JSON, MessagePack, or Thrift message parser. (1405970352123)
-message.event.name=eid
+# Whether the partition file name should also partition by event name/id
+message.partition.byevent=true
+
+# Name of field that contains message/event id/type for JSON, MessagePack, or Thrift message parser. (1405970352123)
+message.event.name=
 
-message.event.mapping.DEFAULT=raw/
+# Default folder name to be appended if partition by event is set to true
+message.event.mapping.DEFAULT=logs/
 
 # Name of field that contains a timestamp, as a date Format, for JSON. (2014-08-07, Jul 23 02:16:57 2005, etc...)
 # Should be used when there is no timestamp in a Long format. Also ignore time zones.
 message.timestamp.input.pattern=yyyy-MM-dd
 
-message.partition.byevent=false
-
 # To enable compression, set this to a valid compression codec implementing 
 # org.apache.hadoop.io.compress.CompressionCodec interface, such as 
 # 'org.apache.hadoop.io.compress.GzipCodec'.

src/main/config/secor.dev.partition.properties

@@ -16,22 +16,22 @@
 include=secor.dev.properties
 
 # Name of the Kafka consumer group.
-secor.kafka.group=secor-me
+secor.kafka.group=secor_partition
 
 # Parser class that extracts s3 partitions from consumed messages.
 secor.message.parser.class=com.pinterest.secor.parser.PatternDateMessageParser
 
 # S3 path where sequence files are stored.
-secor.s3.path=
+secor.s3.path=secor_dev
 
 # Local path where sequence files are stored before they are uploaded to s3.
-secor.local.path=/tmp/secor-me/message_logs/partition
+secor.local.path=/tmp/secor_dev/message_logs/partition
 
 # Port of the Ostrich server.
-ostrich.port=9997
+ostrich.port=9998
 
 # Partition Date Output format. This is used along with PatternDateMessageParser. Defaults to 'yyyy-MM-dd' *New*
-secor.partition.output_dt_format=yyyy-MM-dd
+secor.partition.output_dt_format=dd_MM_yyyy
 
 # Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz.
 # Available placeholders are 
@@ -41,8 +41,7 @@ secor.partition.output_dt_format=yyyy-MM-dd
 # kafkaPartition - The kafka partition
 # fmOffset - First Message offset in the file.
 # randomHex - A 4 character random hex to append to the file name
-# currentTimestamp - Time of upload in epoch format
-# currentTime - Time of upload in HH-mm format
-# currentDate - Time of upload in YYYYMMDD format
+# currentTimestamp - Epoch time
+# currentTime - Time in HH-mm format
 # folder - Folder to use based on message id map lookup
-secor.s3.output_file_pattern={partition}-{currentDate}.json
+secor.s3.output_file_pattern={partition}-{currentTimestamp}.json

src/main/config/secor.dev.properties

@@ -5,7 +5,7 @@ include=secor.common.properties
 ############
 
 # Name of the s3 bucket where log files are stored.
-secor.s3.bucket=lpdev-ekstep
+secor.s3.bucket=
 
 ################
 # END MUST SET #
@@ -18,8 +18,8 @@ zookeeper.quorum=localhost:2181
 
 # Upload policies.
 # 10K
-secor.max.file.size.bytes=1000000000
+secor.max.file.size.bytes=10000
 # 10 seconds
-secor.max.file.age.seconds=300
+secor.max.file.age.seconds=10
 # Compute the max file age for a topic partition using either the oldest or newest file. Defaults to newest
 secor.max.file.age.policy=oldest