kafka+zookeeper+logstash安装


[root&＃64;swarm02 data]# tar -zxvf kafka_2.12-2.2.0.tgz [root&＃64;swarm02 data]# mv kafka_2.12-2.2.0 kafka
[root&＃64;swarm02 data]# ll
total 55696
drwxr-xr-x 6 elk elk 4096 May 5 03:47 elk
drwxr-xr-x 6 root root 83 Mar 9 14:46 kafka
-rw-r--r-- 1 root root 57028557 May 6 10:02 kafka_2.12-2.2.0.tgz
drwxr-xr-x 3 elk elk 78 Apr 30 04:28 skyworking
[root&＃64;swarm02 data]# [root&＃64;swarm02 kafka]# pwd
/data/kafka[root&＃64;swarm02 kafka]# mkdir logsvim /config/server.propertieslog.dirs&＃61;/data/kafka/logs
zookeeper.connect&＃61;localhost:2181启动
/data/kafka/bin/kafka-server-start.sh /data/kafka/config/server.properties &
停止
/data/kafka/bin/kafka-server-stop.sh搭建zookeeper
[root&＃64;swarm02 data]# tar -zxvf zookeeper-3.4.6.tar.gz
[root&＃64;swarm02 data]# mv zookeeper-3.4.6 zookeeper进入Zookeeper的config目录下
拷贝zoo_sample.cfg文件重命名为zoo.cfg&＃xff0c;然后修改dataDir属性[root&＃64;swarm02 data]# mv zookeeper-3.4.6 zookeeper
[root&＃64;swarm02 data]# ll
total 72988
drwxr-xr-x 6 elk elk 4096 May 5 03:47 elk
drwxr-xr-x 7 root root 94 May 6 10:08 kafka
-rw-r--r-- 1 root root 57028557 May 6 10:02 kafka_2.12-2.2.0.tgz
drwxr-xr-x 3 elk elk 78 Apr 30 04:28 skyworking
drwxr-xr-x 10 elk elk 4096 Feb 20 2014 zookeeper
-rw-r--r-- 1 root root 17699306 May 6 10:15 zookeeper-3.4.6.tar.gz
[root&＃64;swarm02 data]# [root&＃64;swarm02 data]# cd zookeeper/[root&＃64;swarm02 zookeeper]# mkdir data
[root&＃64;swarm02 zookeeper]# mkdir log
[root&＃64;swarm02 zookeeper]# ll
total 1532
drwxr-xr-x 2 elk elk 141 Feb 20 2014 bin
-rw-rw-r-- 1 elk elk 82446 Feb 20 2014 build.xml
-rw-rw-r-- 1 elk elk 80776 Feb 20 2014 CHANGES.txt
drwxr-xr-x 2 elk elk 88 May 6 10:31 conf
drwxr-xr-x 10 elk elk 122 Feb 20 2014 contrib
drwxr-xr-x 2 root root 6 May 6 10:33 data
drwxr-xr-x 2 elk elk 4096 Feb 20 2014 dist-maven
drwxr-xr-x 6 elk elk 4096 Feb 20 2014 docs
-rw-rw-r-- 1 elk elk 1953 Feb 20 2014 ivysettings.xml
-rw-rw-r-- 1 elk elk 3375 Feb 20 2014 ivy.xml
drwxr-xr-x 4 elk elk 4096 Feb 20 2014 lib
-rw-rw-r-- 1 elk elk 11358 Feb 20 2014 LICENSE.txt
drwxr-xr-x 2 root root 6 May 6 10:33 log
-rw-rw-r-- 1 elk elk 170 Feb 20 2014 NOTICE.txt
-rw-rw-r-- 1 elk elk 1770 Feb 20 2014 README_packaging.txt
-rw-rw-r-- 1 elk elk 1585 Feb 20 2014 README.txt
drwxr-xr-x 5 elk elk 44 Feb 20 2014 recipes
drwxr-xr-x 8 elk elk 145 Feb 20 2014 src
-rw-rw-r-- 1 elk elk 1340305 Feb 20 2014 zookeeper-3.4.6.jar
-rw-rw-r-- 1 elk elk 836 Feb 20 2014 zookeeper-3.4.6.jar.asc
-rw-rw-r-- 1 elk elk 33 Feb 20 2014 zookeeper-3.4.6.jar.md5
-rw-rw-r-- 1 elk elk 41 Feb 20 2014 zookeeper-3.4.6.jar.sha1
[root&＃64;swarm02 zookeeper]# [root&＃64;swarm02 zookeeper]# cd conf/
[root&＃64;swarm02 conf]# ll
total 12
-rw-rw-r-- 1 elk elk 535 Feb 20 2014 configuration.xsl
-rw-rw-r-- 1 elk elk 2161 Feb 20 2014 log4j.properties
-rw-rw-r-- 1 elk elk 922 Feb 20 2014 zoo_sample.cfg
[root&＃64;swarm02 conf]# cp zoo_sample.cfg zoo.cfg
[root&＃64;swarm02 conf]# ll
total 16
-rw-rw-r-- 1 elk elk 535 Feb 20 2014 configuration.xsl
-rw-rw-r-- 1 elk elk 2161 Feb 20 2014 log4j.properties
-rw-r--r-- 1 root root 922 May 6 10:28 zoo.cfg
-rw-rw-r-- 1 elk elk 922 Feb 20 2014 zoo_sample.cfg
[root&＃64;swarm02 conf]# [root&＃64;swarm02 conf]# vim zoo.cfg
# 数据的存放目录
dataDir&＃61;/data/zookeeper/data
dataLogDir&＃61;/data/zookeeper/log
# 端口&＃xff0c;默认就是2181
clientPort&＃61;2181/data/zookeeper/bin/zkServer.sh start
/data/zookeeper/bin/zkServer.sh stop
/data/zookeeper/bin/zkServer.sh restart
/data/zookeeper/bin/zkServer.sh statusnetstat -tunlp|egrep "(2181|9092)"[root&＃64;swarm02 config]# vim logstash-kaka.confinput {kafka {topics_pattern &＃61;> "cetc-.*" consumer_threads &＃61;> 10decorate_events &＃61;> trueauto_offset_reset &＃61;> "latest"bootstrap_servers &＃61;> "10.0.14.29:9092"codec &＃61;> "json"}
}output {stdout { codec &＃61;> rubydebug }elasticsearch {hosts &＃61;> [ "10.0.14.29:9200" ]index &＃61;> "%{[&＃64;metadata][kafka][topic]}"}}/data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kaka.confnohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kafka.conf &nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/config/logstash-kafka.conf &Sending Logstash logs to /data/elk/logstash/logs which is now configured via log4j2.properties
[2019-05-06T11:16:28,581][WARN ][logstash.config.source.multilocal] Ignoring the &＃39;pipelines.yml&＃39; file because modules or command line options are specified
[2019-05-06T11:16:28,604][INFO ][logstash.runner ] Starting Logstash {"logstash.version"&＃61;>"6.5.4"}
[2019-05-06T11:16:29,567][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path&＃61;>"/data/elk/logstash/conf/logstash-kaka.conf"}
[2019-05-06T11:16:29,577][ERROR][logstash.config.sourceloader] No configuration found in the configured sources.
[2019-05-06T11:16:29,966][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port&＃61;>9600}netstat -tunlp|egrep "(9600)"[root&＃64;swarm02 ~]# /data/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --describe --topic apploghttp://192.168.20.101:9600http://192.168.20.101:9600/_nodehttp://192.168.20.101:9600/_node/statshttp://192.168.20.101:9600/_node/hot_threads# curl http://192.168.20.101:9600/_node/stats?prettyAPI&＃xff1a;Logstash提供了丰富的api来查看Logstash的当前状态