diff --git a/kafka-ingestion/README.md b/kafka-ingestion/README.md index 7fcad36..46dd228 100644 --- a/kafka-ingestion/README.md +++ b/kafka-ingestion/README.md @@ -75,11 +75,11 @@ mysql> desc demo_logs; | Column | Type | Key | Null | Default | Semantic Type | +------------+---------------------+------+------+---------+---------------+ | ip | String | | YES | | FIELD | -| method | String | | YES | | FIELD | +| method | String | PRI | YES | | TAG | +| protocol | String | PRI | YES | | TAG | | path | String | | YES | | FIELD | -| protocol | String | | YES | | FIELD | | user_agent | String | | YES | | FIELD | -| status | UInt32 | | YES | | FIELD | +| status | UInt32 | PRI | YES | | TAG | | size | UInt32 | | YES | | FIELD | | datetime | TimestampNanosecond | PRI | NO | | TIMESTAMP | | timestamp | TimestampNanosecond | | YES | | FIELD | diff --git a/kafka-ingestion/config_data/pipeline.yaml b/kafka-ingestion/config_data/pipeline.yaml index c191f67..5634fac 100644 --- a/kafka-ingestion/config_data/pipeline.yaml +++ b/kafka-ingestion/config_data/pipeline.yaml @@ -18,13 +18,22 @@ processors: transform: - fields: - ip + type: string + - fields: - method - - path - protocol + type: string + index: tag + - fields: + - path - user_agent type: string + index: fulltext - fields: - status + type: uint32 + index: tag + - fields: - size type: uint32 - fields: