-
Notifications
You must be signed in to change notification settings - Fork 0
/
kafka_commands.properties
86 lines (43 loc) · 2.92 KB
/
kafka_commands.properties
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#zookeeper start
zookeeper-server-start.bat config/zookeeper.properties
runs on port 2181
#kafka start
kafka-server-start.bat config/server.properties
runs on port 9092
#kafka_topics
kafka_topics.bat
kafka-topics --zookeeper 127.0.0.1:2181 --topic important_tweets --create --partitions 3 --replication-factor 1
--cannot create replication-factor more than no of brokers
kafka-topics.bat --zookeeper 127.0.0.1:2181 --list
kafka-topics.bat --zookeeper 127.0.0.1:2181 --topic first_topic --describe
#Kafka_producer
kafka-console-producer.bat --broker-list 127.0.0.1:9092 --topic first_topic
kafka-console-producer.bat --broker-list 127.0.0.1:9092 --topic first_topic --producer-property acks=all
kafka-console-producer.bat --broker-list 127.0.0.1:9092 --topic new_topic
#multi_broker
kafka-console-producer.bat --broker-list 127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094 --topic many-brokers
--used in production because when 1 broker is down, we can still connect to other active brokers and connect to the cluster
#deafult_partitions
--always create topic before producing
--set num.partitions=3 to always set min partitions to 3
#Kafka_consumer
#if consumer group is not specified, then default consumer group is created
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic first_topic
#to read all message from beginning
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic twitter_status_connect --from-beginning
#Kafka_consumers_group
kafka-console-consumer --bootstrap-server 127.0.0.1:9092 --topic first_topic --group my_first_application
kafka-consumer-groups --bootstrap-server localhost:9092 --list
kafka-consumer-groups --bootstrap-server 127.0.0.1:9092 --describe --group my_first_application
#the above commands specifies the topics,partitions,offsets, lag, consumers, host etc
#if the lag is 0, it means all messages are read. if there is a lag on the consumer group, then some messgaes are not read
#Reset Offsets
#when a reset is done using reset offsets, the messages will be read from the beginning
kafka-consumer-groups --bootstrap-server 127.0.0.1:9092 --group my_first_application --reset-offsets --to-earliest --execute --topic first_topic
#when a reset is done using shift by -2, the messages will be read from 2 offsets before
kafka-consumer-groups --bootstrap-server 127.0.0.1:9092 --group my_first_application --reset-offsets --shift-by -2 --execute --topic first_topic
kafka-consumer-groups --bootstrap-server 127.0.0.1:9092 --group kafka-elasticsearch-application --reset-offsets --to-earliest --execute --topic twitter_tweets
#docker
kafka-topics --zookeeper 192.168.99.100:2181 --topic test --create --partitions 3 --replication-factor 1
kafka-console-producer --broker-list 192.168.99.100:9092,192.168.99.100:9093,192.168.99.100:9094 --topic test
kafka-console-consumer --bootstrap-server 192.168.99.100:9092 --topic single-docker --group --from-beginning