AWS linux kafka 초기 설정

./bin/kafka-server-start.sh config/server.properties

java 설치

yum list java*
sudo yum install java-11-amazon-corretto.x86_64

 

서버 설정

sudo vi /etc/hosts

15.165.47.225 kafka-01
43.203.183.70 kafka-02
43.203.163.54 kafka-03

 

zookeeper 설치

Apache ZooKeeper

Apache ZooKeeper

wget https://dlcdn.apache.org/zookeeper/zookeeper-3.8.4/apache-zookeeper-3.8.4-bin.tar.gz
tar zxf apache-zookeeper-3.8.4-bin.tar.gz

 

zookeeper 환경설정

cd apache-zookeeper-3.8.4-bin/conf
cp zoo_sample.cfg zoo.cfg
vi zoo.cfg

/data/app_logs/zookeeper

server.1=kafka-01:2888:3888
server.2=kafka-02:2888:3888
server.3=kafka-03:2888:3888

sudo mkdir -p /data/app_logs/zookeeper
udo chmod -R 777 /data

# kafka-01 서버에서 실행
echo 1 > /data/app_logs/zookeeper/myid
# kafka-02 서버에서 실행
echo 2 > /data/app_logs/zookeeper/myid
# kafka-03 서버에서 실행
echo 3 > /data/app_logs/zookeeper/myid

zookeeper 실행

bin/zkServer.sh start

./bin/zookeeper-server-start.sh config/zookeeper.properties

 

kafka 설치

wget https://downloads.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz
tar zxf kafka_2.13-3.8.0.tgz

 

kafka 환경 변수 설정

nano .bashrc
PATH="$PATH:~/kafka_2.13-3.8.0/bin"

 

kafka 서버 설정

vi kafka_2.13-3.8.0/config/server.properties

broker.id=1
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://kafka-1:9092
zookeeper.connect=kafka-01:2181,kafka-02:2181,kafka-03:2181

 

카프카 실행(백그라운드)

bin/kafka-server-start.sh -daemon config/server.properties

 

카프카 실행

./bin/kafka-server-start.sh config/server.properties

 

카프카 중지

bin/kafka-server-stop.sh

 

producer

./bin/kafka-console-producer.sh --topic test-topic --bootstrap-server localhost:9092

 

./bin/kafka-console-consumer.sh --topic test-topic --bootstrap-server localhost:9092

consumer