{"id":526,"date":"2018-01-24T18:41:00","date_gmt":"2018-01-24T09:41:00","guid":{"rendered":"https:\/\/wp.zassoul.com\/?p=526"},"modified":"2018-01-24T18:41:00","modified_gmt":"2018-01-24T09:41:00","slug":"logstash%e3%81%ae%e3%83%9e%e3%83%ab%e3%83%81%e3%83%91%e3%82%a4%e3%83%97%e3%83%a9%e3%82%a4%e3%83%b3%e8%a8%ad%e5%ae%9a","status":"publish","type":"post","link":"https:\/\/wp.zassoul.com\/?p=526","title":{"rendered":"Logstash\u306e\u30de\u30eb\u30c1\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u8a2d\u5b9a"},"content":{"rendered":"

Filebeats\u304b\u3089\u306e\u5165\u529b\u3068\u30ed\u30fc\u30ab\u30eb\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u3093\u3067\u306e\u51e6\u7406\u3092\u5206\u3051\u308b\u305f\u3081\u306b\u30de\u30eb\u30c1\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306e\u8a2d\u5b9a\u3092\u3059\u308b\u3002<\/p>\n\n\n\n\n
\"blog-pipeline-log.jpg\"<\/td>\n<\/tr>\n
\u516c\u5f0f\u30b5\u30a4\u30c8(Introducing Multiple Pipelines in Logstash)\u3088\u308a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

https:\/\/www.elastic.co\/blog\/logstash-multiple-pipelines<\/a>
https:\/\/www.elastic.co\/guide\/en\/logstash\/master\/multiple-pipelines.html<\/a><\/p>\n

\u4ee5\u524d\u4f5c\u6210\u3057\u305fFilebeats\u306e\u8a2d\u5b9a\u30d5\u30a1\u30a4\u30eb\u3092\u8ee2\u7528\u3057\u3066, CSV\u30d5\u30a1\u30a4\u30eb\u7528\u306e\u8a2d\u5b9a\u30d5\u30a1\u30a4\u30eb\u3092\u4f5c\u6210\u3057\u3066, \u305d\u308c\u305e\u308c\u7570\u306a\u308b\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u3067\u51e6\u7406\u3059\u308b\u8a2d\u5b9a\u3092\u884c\u3046\u3002
CSV\u30d5\u30a1\u30a4\u30eb\u3067\u4f7f\u3046\u306e\u306f\u9577\u5e74\u30a8\u30af\u30bb\u30eb\u3067\u7ba1\u7406\u3057\u3066\u30b0\u30e9\u30d5\u5316\u3057\u3066\u3044\u305f\u5149\u71b1\u8cbb\u306e\u30c7\u30fc\u30bf\u3092\u4f7f\u3046\uff08\u7b11\uff09\u3002<\/p>\n

\u624b\u9806<\/b>
1. \/etc\/logstash\/pipelines.yml \u4f5c\u6210
2. filebeats\u306eConfig\uff08\u65e2\u5b58\u8ee2\u7528\uff09\u3068CSV\u7528\u306eConfig\u3092\u4f5c\u6210
3. index Pattern\u4f5c\u6210
4. \u30b0\u30e9\u30d5\u4f5c\u6210<\/p>\n

1. pipelines.yml \u4f5c\u6210<\/h4>\n

pipelines.yml\u306b\u306f\u5404\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306e\u8a2d\u5b9a\u3092\u8a18\u8ff0\u3059\u308b\u3002\u3053\u3053\u306b\u8a18\u8ff0\u3055\u308c\u306a\u304b\u3063\u305f\u8a2d\u5b9a\u306flogstash.yml\u3092\u53c2\u7167\u3057\u3066\u305d\u308c\u306b\u5f93\u3046\u3068\u306e\u3053\u3068\u3002
\u30fbpipelines.yml \u306b\u306f\u500b\u5225\u8a2d\u5b9a
\u30fblogstash.yml \u306b\u306f\u5171\u901a\u8a2d\u5b9a
\u3068\u3044\u3046\u4f4f\u307f\u5206\u3051\u3089\u3057\u3044\u3002(\u591a\u5206)<\/p>\n

vi \/etc\/logstash\/pipelines.yml<\/p>\n

# For beats
- pipeline.id: filebeat
  pipeline.workers: 2
  pipeline.batch.size: 125
  pipeline.batch.delay: 5
  config.reload.automatic: true
  config.reload.interval: 5s
  path.config: \"\/etc\/logstash\/pipeconf.d\/beats.conf\"

# For csv
- pipeline.id: csvfile
  pipeline.workers: 1
  pipeline.batch.size: 125
  pipeline.batch.delay: 5
  config.reload.automatic: true
  config.reload.interval: 5s
  path.config: \"\/etc\/logstash\/pipeconf.d\/csv.conf\"
<\/pre>\n
\u3053\u308c\u306b\u4f34\u3063\u3066 \/etc\/logstash\/logstash.yml \u3092\u4ee5\u4e0b\u306e\u3088\u3046\u306b\u5909\u66f4\u3057\u305f\u3002
vi \/etc\/logstash\/logstash.yml<\/p>\n
path.data: \/var\/lib\/logstash
log.level: info
path.logs: \/var\/log\/logstash
<\/pre>\n
\u6700\u4f4e\u9650\u306e\u3082\u306e\u306b\u7d5e\u3063\u305f\u306e\u3067\u8a2d\u5b9a\u5024\u306fpipelines.yml\u3078\u6301\u3063\u3066\u3044\u3063\u305f\u3002<\/p>\n
2. csv\u7528\u8a2d\u5b9a\u30d5\u30a1\u30a4\u30eb\u4f5c\u6210<\/h4>\n
csv\u30d5\u30a1\u30a4\u30eb\u306f\u6b21\u306e\u3088\u3046\u306a\u8981\u7d20\u306b\u306a\u3063\u3066\u3044\u308b\u3002
\u300cYYYYM,\u96fb\u6c17\u4ee3,\u6c34\u9053\u4ee3,\u30ac\u30b9\u4ee3\u300d
\u5b9f\u969b\u306e\u30c7\u30fc\u30bf\u306f\u3053\u3093\u306a\u611f\u3058\u3002
\u300c201712,9308,11765,10018\u300d<\/p>\n
logstash\u306flogstash\u30e6\u30fc\u30b6\u3067\u8d77\u52d5\u3059\u308b\u306e\u3067, \u30c7\u30fc\u30bf\u306e\u914d\u7f6e\u5834\u6240\u306f\u53c2\u7167\u3067\u304d\u308b\u3068\u3053\u308d\u3078\u914d\u7f6e\u3059\u308b\u3002(\u79c1\u306f\u30e6\u30fc\u30b6\u306e\u30db\u30fc\u30e0\u306b\u304a\u3044\u3066read\u6a29\u9650\u3042\u308b\u304b\u3089\u5927\u4e08\u592b\u3060\u308d\u3046\u3068\u601d\u3063\u3066\u3044\u305f\u3089\u30cf\u30de\u3063\u305f\u3002)<\/div>\n
\u5404\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u7528\u306e\u8a2d\u5b9a\u30d5\u30a1\u30a4\u30eb\u306f \/etc\/logstash \u306b pipeconf.d \u3068\u3044\u3046\u30c7\u30a3\u30ec\u30af\u30c8\u30ea\u3092\u65b0\u305f\u306b\u4f5c\u6210\u3057\u3066\u305d\u3053\u306b\u914d\u7f6e\u3057\u305f\u3002<\/div>\n
vi \/etc\/logstash\/pipeconf.d\/csv.conf<\/div>\n
<\/p>\n
input {
  file {
    path => \"\/tmp\/logstash\/kounetsu.csv\"
    start_position => \"beginning\"
  }
}

filter {
  csv {
    columns => [\"date\", \"elec\", \"water\", \"gus\"]
    skip_empty_columns => true
    convert => {
      \"elec\" => \"integer\"
      \"water\" => \"integer\"
      \"gus\" => \"integer\"
    }
  }
  date {
    match => [ \"date\", \"yyyyM\"]
  }
}

output {
  elasticsearch {
    hosts => [ \"http:\/\/172.16.10.50:9200\" ]
    index => \"kounetsu\"
  }
}
<\/pre>\n
<\/p>\n
3. index pattern\u4f5c\u6210<\/h4>\n
elasticsearch\u3078”kounetsu”\u3068\u3044\u3046\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3067\u6e21\u3057\u3066\u3044\u308b\u306e\u3067 kibana \u3067\u3053\u308c\u3092\u6307\u5b9a\u3057\u3066\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u30d1\u30bf\u30fc\u30f3\u3092\u4f5c\u6210\u3059\u308b\u3002<\/p>\n
<\/a><\/div>\n
<\/p>\n
<\/div>\n
4. \u30b0\u30e9\u30d5\u4f5c\u6210<\/h4>\n
\u3053\u3053\u306f\u304a\u597d\u307f\u3067\u3068\u306a\u308a\u307e\u3059\u304c\u3002\u3002\u3002<\/p>\n
<\/a><\/div>\n
\u5b8c\u6210\u3002<\/p><\/div>\n
\n
\u3067\u304d\u308b\u307e\u3067\u7d50\u69cb\u30cf\u30de\u3063\u3066\u3044\u305f\u306e\u3067\u3053\u308c\u3092\u30dd\u30c1\u3063\u305f\u3002 \u7121\u99c4\u306b\u306f\u306a\u3089\u306a\u3044\u3088\u306d\u30fb\u30fb\u30fb\u3002 <\/a>\"\"<\/p>\n","protected":false},"excerpt":{"rendered":"
Filebeats\u304b\u3089\u306e\u5165\u529b\u3068\u30ed\u30fc\u30ab\u30eb\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u8aad\u307f\u8fbc\u3093\u3067\u306e\u51e6\u7406\u3092\u5206\u3051\u308b\u305f\u3081\u306b\u30de\u30eb\u30c1\u30d1\u30a4\u30d7\u30e9\u30a4\u30f3\u306e\u8a2d\u5b9a\u3092\u3059\u308b\u3002 \u516c\u5f0f\u30b5\u30a4\u30c8(Introducing Multiple Pipelines in Logstash)\u3088\u308a h\u2026 \u7d9a\u304d\u3092\u8aad\u3080 »<\/a><\/span><\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"","ping_status":"","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[51,55,56,8],"tags":[],"class_list":["post-526","post","type-post","status-publish","format-standard","hentry","category-elasticsearch","category-kibana","category-logstash","category-8"],"_links":{"self":[{"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=\/wp\/v2\/posts\/526","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=526"}],"version-history":[{"count":0,"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=\/wp\/v2\/posts\/526\/revisions"}],"wp:attachment":[{"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=526"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=526"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/wp.zassoul.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=526"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}