Configuration

back

0. assumption

(a) sample source

<source>
    type tail
    format /^(?<time>[^ ]* [^ ]* [^ ]* [^ ]*) (?<key1>[^ ]*) (?<key2>[^ ]*) (?<key3>[^ ]*) (?<value1>[^ ]*)$/
    time_format %a %b %e %H:%M:%S
    path /var/log/something.log
    tag aggr_hostneme
</source>

(b) interval

we assume aggregate per minute.

(c) metrics

    <metrics>
        name one_key
        partition_by m
        each_key key1
    </metrics>
    <metrics>
        name two_keys
        partition_by m
        each_key key2,key3
        value_key value1
        type float
    </metrics>

this configuration means aggregate 2 metrics:

• count(*) group by key1,
• sum(value) group by key2, key3.

1. aggregation_forward

(a) for start point

<match aggr_**>
    type aggregation_forward
    flush_interval 1m
    send_timeout 60s
    recover_wait 10s
    heartbeat_interval 1s
    hard_timeout 60s
    <metrics>
        name one_key
        partition_by m
        each_key key1
    </metrics>
    <metrics>
        name two_keys
        partition_by m
        each_key key2,key3
        value_key value1
        type float
    </metrics>
    <server>
        name host1
        host host1
        port 24224
    </server>
    <server>
        name host2
        host host2
        port 24224
    </server>
    <secondary>
        type file
        path /home/doryokujin/forward-failed
    </secondary>
</match>

(b) for relay point

<match aggr_**>
    type aggregation_forward
    flush_interval 1m
    send_timeout 60s
    recover_wait 10s
    heartbeat_interval 1s
    hard_timeout 60s
    <server>
        name host3
        host host3
        port 24224
    </server>
    <server>
        name host4
        host host4
        port 24224
    </server>
    <secondary>
        type file
        path /home/doryokujin/forward-failed
    </secondary>
</match>

aggregation_mongo

(a) for start point

<match aggr_**>
    type aggregation_mongo
    time_slice_wait 1m
    time_slice_format %Y%m%d%H%M
    buffer_path /home/doryokujin/buffer.log
    database test
    collection aggr
    host localhost
    port 27017
    <metrics>
        name one_key
        partition_by m
        each_key key1
    </metrics>
    <metrics>
        name two_keys
        partition_by m
        each_key key2,key3
        value_key value1
        type float
    </metrics>
</match>

(b) for end point

<match aggr_**>
    type aggregation_mongo
    time_slice_wait 1m
    time_slice_format %Y%m%d%H%M
    buffer_path /home/doryokujin/buffer.log
    database test
    collection aggr
    host localhost
    port 27017
</match>

aggregation_redis

<match aggr_**>
    type aggregation_mongo
    time_slice_wait 1m
    time_slice_format %Y%m%d%H%M
    buffer_path /home/doryokujin/buffer.log
    host localhost
    port 6379
    db 0
    # expire 3600
</match>