Build Zookeeper and Kafka clusters

Zookeeper and Kafka cluster system environment configuration

Configure IP

ssh [email protected] "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"
ssh [email protected] "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"
ssh [email protected] "rm -rf /etc/machine-id; systemd-machine-id-setup;reboot"

ssh [email protected] "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"
ssh [email protected] "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"
ssh [email protected] "nmcli con delete uuid d1141403-18c6-3149-907c-ed5f09663a7f;nmcli con add type ethernet ifname ens160 con-name ens160;nmcli con up ens160"

ssh [email protected] "nmcli con mod ens160  192.168.1.61/24; nmcli con mod ens160   192.168.1.1; nmcli con mod ens160  manual; nmcli con mod ens160  "8.8.8.8"; nmcli con up ens160"
ssh [email protected] "nmcli con mod ens160  192.168.1.62/24; nmcli con mod ens160   192.168.1.1; nmcli con mod ens160  manual; nmcli con mod ens160  "8.8.8.8"; nmcli con up ens160"
ssh [email protected] "nmcli con mod ens160  192.168.1.63/24; nmcli con mod ens160   192.168.1.1; nmcli con mod ens160  manual; nmcli con mod ens160  "8.8.8.8"; nmcli con up ens160"

ssh [email protected] "nmcli con mod ens160  fc00::61/8; nmcli con up ens160"
ssh [email protected] "nmcli con mod ens160  fc00::62/8; nmcli con up ens160"
ssh [email protected] "nmcli con mod ens160  fc00::63/8; nmcli con up ens160"


[root@localhost ~]#  cat /etc/NetworkManager/system-connections/ 
[connection]
id=ens160
uuid=94c63fdf-cd5a-427c-9846-5a447de2a4f5
type=ethernet
interface-name=ens160
timestamp=1744436596

[ethernet]

[ipv4]
address1=192.168.1.61/24,192.168.1.1
dns=192.168.1.99;
method=manual

[ipv6]
addr-gen-mode=default
address1=fc00::61/8
method=auto

[proxy]

Set the host name

hostnamectl set-hostname zk-1
hostnamectl set-hostname zk-2
hostnamectl set-hostname zk-3

Turn off firewall, selinux

# Turn off the firewall
 systemctl disable --now firewalld
 # Close selinux
 setenforce 0
 sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

Install JDK

# Check whether jdk exists, and if installed, uninstall jdk first
 rpm -qa|grep jdk
 # Download jdk
 /java/technologies/downloads/#java8

 # Unzip jdk
 tar -xvf
 # Move to the installation directory
 mv jdk1.8.0_441/ /usr/local/

 # Edit environment variables
 vim /etc/profile

 export JAVA_HOME=/usr/local/jdk1.8.0_441
 export CLASSPATH=$JAVA_HOME/lib/:$JAVA_HOME/lib/
 export PATH=$PATH:$JAVA_HOME/bin

 # Refresh environment variables
 source /etc/profile

 # View version
 [root@zk-1 ~]# java -version
 java version "1.8.0_441"
 Java(TM) SE Runtime Environment (build 1.8.0_441-b07)
 Java HotSpot(TM) 64-Bit Server VM (build 25.441-b07, mixed mode)

Install Zookeeper cluster

# Download Zookeeper
 # /dist/zookeeper/

 wget /dist/zookeeper/zookeeper-3.9.3/apache-zookeeper-3.9.9.

 # Create an application directory
 mkdir -vp /app/zookeeper-cluster
 cd /app/zookeeper-cluster

 # Unzip installation
 mv /root/apache-zookeeper-3.9. .
 tar xvf apache-zookeeper-3.9.
 mv apache-zookeeper-3.9.3-bin zk

Modify zookeeper configuration

# Create a directory to store data and logs
 cd zk
 mkdir data logs

 # Copy configuration example
 cd conf
 cp zoo_sample.cfg


 # Modify the configuration file
 vi

 # Set logs and storage directories
 dataDir=/app/zookeeper-cluster/zk/data
 dataLogDir=/app/zookeeper-cluster/zk/logs
 # Add node information
 server.1=192.168.1.61:2888:3888
 server.2=192.168.1.62:2888:3888
 server.3=192.168.1.63:2888:3888
 # Add sasl
 authProvider.1=
 jaasLoginRenew=3600000
 requireClientAuthScheme=sasl
 =true

My full configuration

[root@zk-1 conf]# cat  | grep -Ev '^$|#'
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/app/zookeeper-cluster/zk/data
dataLogDir=/app/zookeeper-cluster/zk/logs
clientPort=2181
server.1=192.168.1.61:2888:3888
server.2=192.168.1.62:2888:3888
server.3=192.168.1.63:2888:3888
authProvider.1=
jaasLoginRenew=3600000
requireClientAuthScheme=sasl
=true

Set up sasl authentication

# Set sasl authentication
 cat <<EOF | tee /app/zookeeper-cluster/zk/conf/zk_jaas.conf
 Server {
     required
    user_cby="Cby123..";
 };

 Client {
         required
        username="cby"
        password="Cby123..";
 };
 EOF

 # Modify zookeeper environment variables
 # Add variables at the bottom
 vim
 export JVMFLAGS="-=/app/zookeeper-cluster/zk/conf/zk_jaas.conf ${JVMFLAGS}"

 #My results
 [root@zk-1 bin]# cat | grep JVMFLAGS
 export SERVER_JVMFLAGS="-Xmx${ZK_SERVER_HEAP}m $SERVER_JVMFLAGS"
 export CLIENT_JVMFLAGS="-Xmx${ZK_CLIENT_HEAP}m $CLIENT_JVMFLAGS"
 export JVMFLAGS="-=/app/zookeeper-cluster/zk/conf/zk_jaas.conf ${JVMFLAGS}"

Set up startup mode

# Create myid file in the data directory

 #Configuration on Node 1
 echo 1 >/app/zookeeper-cluster/zk/data/myid
 #Configuration on Node 2
 echo 2 >/app/zookeeper-cluster/zk/data/myid
 #Configuration on Node 3
 echo 3 >/app/zookeeper-cluster/zk/data/myid

 # Create a system startup file
 cat <<EOF | tee /usr/lib/systemd/system/
 [Unit]
 Description=zookeeper
 After=

 [Service]
 Type=forking
 Environment=JAVA_HOME=/usr/local/jdk1.8.0_441
 ExecStart=/app/zookeeper-cluster/zk/bin/ start
 ExecStop=/app/zookeeper-cluster/zk/bin/ stop
 PIDFile=/app/zookeeper-cluster/zk/data/zookeeper_server.pid
 KillMode=none
 User=root
 Group=root
 Restart=on-failure
 [Install]
 WantedBy=
 EOF

Start and stop

# Reload
 systemctl daemon-reload
 # start up
 systemctl start zookeeper
 # stop
 systemctl stop zookeeper
 # Set up power-on and start
 systemctl enable --now zookeeper
 # Check the status
 systemctl status zookeeper

Usage test

# Log in to each node to set sasl
 /app/zookeeper-cluster/zk/bin/ -server 192.168.1.61:2181
 /app/zookeeper-cluster/zk/bin/ -server 192.168.1.62:2181
 /app/zookeeper-cluster/zk/bin/ -server 192.168.1.63:2181

 # Set Acl read and write permissions
 setAcl / sasl:cby:crdwa

 # Create directory and Acl verification command:
 addauth digest super:cby
 create /cby
 getAcl /

 ......slightly.....
 [zk: 192.168.1.63:2181(CONNECTED) 0] setAcl / sasl:cby:crdwa
 [zk: 192.168.1.63:2181(CONNECTED) 1]
 [zk: 192.168.1.63:2181(CONNECTED) 1] addauth digest super:cby
 [zk: 192.168.1.63:2181(CONNECTED) 2] create /cby
 Node already exists: /cby
 [zk: 192.168.1.63:2181(CONNECTED) 3] getAcl /
 'sasl,'cby
 : cdrwa
 [zk: 192.168.1.63:2181(CONNECTED) 4]

Install Kafka cluster

# Download Kafka
 # /downloads
 # wget /kafka/3.9.0/kafka_2.13-3.9.
 wget /apache/kafka/3.9.0/kafka_2.13-3.9.

 # Create an application directory
 mkdir -vp /app/kafka-cluster
 cd /app/kafka-cluster

 # Unzip installation
 mv /root/kafka_2.13-3.9. .
 tar xvf kafka_2.13-3.9.
 mv kafka_2.13-3.9.0 kafka
 mkdir -p /app/kafka-cluster/kafka/kafka-logs

Modify configuration

# Modify the configuration
 cd /app/kafka-cluster/kafka/config
 vim

 # Modify items: , , listeners, and add the following configurations
 # The three units are set to inconsistent
 =SASL_PLAINTEXT
 =PLAIN
 =PLAIN
 =
 =false
 =User:cby

My configuration

#My configuration
 [root@zk-1 config]# cat | grep -Ev '^$|#'
 =1
 listeners=SASL_PLAINTEXT://:9092
 =SASL_PLAINTEXT://192.168.1.61:9092
 =3
 =8
 =102400
 =102400
 =104857600
 =/app/kafka-cluster/kafka/kafka-logs
 =1
 =1
 =1
 =1
 =1
 =168
 =300000
 =192.168.1.61:2181,192.168.1.62:2181,192.168.1.63:2181
 =18000
 =0
 =SASL_PLAINTEXT
 =PLAIN
 =PLAIN
 =
 =false
 =User:cby

 # Modify the configuration
 cd /app/kafka-cluster/kafka/config
 vim

 [root@zk-1 config]# cat | grep -Ev '^$|#'
 =192.168.1.61:9092,192.168.1.62:9092,192.168.1.63:9092
 =none

Service configuration authentication

# Server configuration login authentication
 cat <<EOF | tee /app/kafka-cluster/kafka/config/kafka_server_jaas.conf
 KafkaServer {
      required
     username="cby"
     password="Cby123.."
     user_cby="Cby123..";
 };
 Client {
      required
     username="cby"
     password="Cby123..";
 };
 EOF

Customer configuration certification

# Client configuration login authentication
 cat <<EOF | tee /app/kafka-cluster/kafka/config/kafka_client_jaas.conf
 KafkaClient {
         required
        username="cby"
        password="Cby123..";

 };
 EOF

Configure Start Authentication

# Modify the script and add -=/app/kafka-cluster/kafka/config/kafka_server_jaas.conf
 vim
 [root@zk-3 bin]# cat | grep -Ev '^$|#'
 Then
	 echo "USAGE: $0 [-daemon] [--override property=value]*"
	 exit 1
 fi
 base_dir=$(dirname $0)
 if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
     export KAFKA_LOG4J_OPTS="-=file:$base_dir/../config/"
 fi
 if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
     export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G -=/app/kafka-cluster/kafka/config/kafka_server_jaas.conf"
 fi
 EXTRA_ARGS=${EXTRA_ARGS-'-name kafkaServer -loggc'}
 COMMAND=$1
 case $COMMAND in
   -daemon)
     EXTRA_ARGS="-daemon "$EXTRA_ARGS
     shift
     ;;
   *)
     ;;
 esac
 exec $base_dir/ $EXTRA_ARGS "$@"
 [root@zk-3 bin]#

Configure production and consumption certification

# Configure production and consumer login verification
 # Add -=/app/kafka-cluster/kafka/config/kafka_client_jaas.conf after export KAFKA_HEAP_OPTS="-Xmx512M

 vim
 [root@zk-1 bin]# cat | grep -Ev '^$|#'
 if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
     export KAFKA_HEAP_OPTS="-Xmx512M -=/app/kafka-cluster/kafka/config/kafka_client_jaas.conf"
 fi
 exec $(dirname $0)/ "$@"
 [root@zk-1 bin]#

 vim
 [root@zk-1 bin]# cat | grep -Ev '^$|#'
 if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
     export KAFKA_HEAP_OPTS="-Xmx512M -=/app/kafka-cluster/kafka/config/kafka_client_jaas.conf"
 fi
 exec $(dirname $0)/ "$@"
 [root@zk-1 bin]#

Configure Start Stop

# Create a system startup file
 cat <<EOF | tee /usr/lib/systemd/system/
 [Unit]
 Description=kafka-node01
 After=

 [Service]
 Type=simple
 Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/jdk1.8.0_441/bin"
 User=root
 Group=root
 LimitNOFILE=100000
 ExecStart=/app/kafka-cluster/kafka/bin/ /app/kafka-cluster/kafka/config/
 ExecStop=/app/kafka-cluster/kafka/bin/
 Restart=on-failure

 [Install]
 WantedBy=
 EOF

Start and stop

# Reload
 systemctl daemon-reload
 # start up
 systemctl start kafka
 # stop
 systemctl stop kafka
 # Set up power-on and start
 systemctl enable --now kafka
 # Check the status
 systemctl status kafka

Configure account password

#Because SSL is configured, you need to configure encrypted authentication files
 cat <<EOF | tee /app/kafka-cluster/kafka/config/
 =SASL_PLAINTEXT
 =PLAIN
 = required username="cby" password="Cby123..";
 EOF

Test usage

# Create a topic
 ./ --create --topic cby --replication-factor 3 --partitions 3 --bootstrap-server 192.168.1.61:9092,192.168.1.62:9092,192.168.1.63:9092 --command-config ../config/

 # View topic
 ./ --list --bootstrap-server 192.168.1.61:9092,192.168.1.62:9092,192.168.1.63:9092 --command-config ../config/

 # Send a message
 ./ --broker-list 192.168.1.61:9092,192.168.1.62:9092,192.168.1.63:9092 --topic cby -- ../config/
 >123
 >321

 # View message
 ./ --bootstrap-server 192.168.1.61:9092,192.168.1.62:9092,192.168.1.63:9092 --topic cby --from-beginning -- ../config/
 123
 321

 # Remove topic
 ./ --delete --topic cby --bootstrap-server 192.168.1.61:9092,192.168.1.62:9092,192.168.1.63:9092 --command-config ../config/

about

CSDN, GitHub, Zhihu, Open Source China, Sifu, Nuggets, Jianshu, Huawei Cloud, Alibaba Cloud, Tencent Cloud, Bilibili, Toutiao, Sina Weibo, Personal Blog

"Xiao Chen Operation and Maintenance" can be searched on the entire network