Location>code7788 >text

flink + iceberg quick build guide

Popularity：588 ℃/2024-08-18 15:03:08

flink + iceberg quick build

the environment includes:

  • minio
  • iceberg
  • flink

Centos replaces tencent's yum sources.

Backing up old system configuration files

mv /etc// /etc//

Get the corresponding version to the /etc// directory.

List of source configurations by version

CentOS7 
wget -O /etc// /repo/centos7_base.repo
CentOS8 
wget -O /etc// /repo/centos8_base.repo

Updating the cache 

yum clean all
yum makecache

Centos install Java environment (flink use)

wget --no-check-certificate /java/jdk/8u202-b08/

mkdir /usr/local/java/
tar -zxvf  -C /usr/local/java

echo "export JAVA_HOME=/usr/local/java/jdk1.8.0_202" >> /etc/profile
echo "export JRE_HOME=${JAVA_HOME}/jre" >> /etc/profile
echo "export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib" >> /etc/profile
echo "export PATH=${JAVA_HOME}/bin:$PATH" >> /etc/profile

source /etc/profile
ln -s /usr/local/java/jdk1.8.0_202/bin/java /usr/bin/java

java -version

Centos install Docker & Docker Compose

First, To install Docker Engine, you need a maintained version of CentOS 7 or 8.
Archived versions aren't supported or tested.

sudo yum install -y python3-pip yum-utils device-mapper-persistent-data lvm2
sudo yum-config-manager --add-repo /docker-ce/linux/centos/
sudo yum install docker-ce
sudo systemctl start docker

update docker daemon..

sudo mkdir -p /etc/docker
sudo tee /etc/docker/ <<-'EOF'
{
  "registry-mirrors": [
    "/",
	""
  ]
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker

install docker compose..

sudo curl -L "/docker-toolbox/linux/compose/1.21.2/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
docker-compose --version

install Minio

Use the following commands to download and install the latest version of the stable MinIO binaries, and set up the$PATH:

wget /server/minio/release/linux-amd64/minio
chmod +x minio
sudo mv minio /usr/local/bin/

Setting up the minio linux service

sudo tee /etc/systemd/system/ <<-'EOF'
[Unit]
Description=Minio Service

[Service]
Environment="MINIO_ROOT_USER=admin"
Environment="MINIO_ROOT_PASSWORD=password"
ExecStart=/usr/local/bin/minio server /mnt/minio --console-address ":9001"
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s QUIT $MAINPID
StandardOutput=/mnt/minio/logs/
PrivateTmp=true

[Install]
WantedBy=
EOF

Start the minio linux service

systemctl start minio
systemctl enable minio

install iceberg in docker

save the yaml below into a file named :

wget /server/minio/release/linux-amd64/minio
version: "3"

services:
  rest:
    image: tabulario/iceberg-rest:1.5.0
    container_name: iceberg-rest
    networks:
      iceberg_net:
    ports:
      - 8181:8181
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - CATALOG_WAREHOUSE=s3://warehouse/
      - CATALOG_IO__IMPL=.s3.S3FileIO
      - CATALOG_S3_ENDPOINT=<Replacement of localminioservice address:9000>
  mc:
    image: minio/mc
    container_name: mc
    networks:
      iceberg_net:
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://<Replacement of localminioservice address:9000> admin password) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc mb minio/warehouse;
      /usr/bin/mc policy set public minio/warehouse;
      tail -f /dev/null
      "
networks:
  iceberg_net:

Next, start up the docker containers with this command:

docker-compose up -d

install flink

download jdk filnk 1.18.1

wget /apache/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.

tar zxvf flink-1.18.1-bin-scala_2. -C /opt/

download dll ，copy to flink-1.18.1/lib folder

wget /maven2/org/apache/iceberg/iceberg-aws-bundle/1.5.2/iceberg-aws-bundle-1.5.
wget ttps:///maven2/org/apache/iceberg/iceberg-flink-runtime-1.18/1.5.2/iceberg-flink-runtime-1.18-1.5.
wget /maven2/org/apache/flink/flink-connector-jdbc/3.1.2-1.18/flink-connector-jdbc-3.1.2-1.
wget /maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.
wget /maven2/org/apache/flink/flink-s3-fs-hadoop/1.18.0/flink-s3-fs-hadoop-1.18.

cp ./*.jar /opt/flink-1.18.1/lib/

start flink

cd /opt/flink-1.18.1/

bash ./bin/

flink create icberg table

bash flink client:

cd /opt/flink-1.18.1/

bash ./bin/

SET '-mode' = 'tableau';
SET '-mode' = 'batch';

Create Iceberg Catalog

CREATE CATALOG iceberg WITH (
  'type'='iceberg',
  'catalog-type'='rest',
  'uri'='http://rest:8181/',
  ''='http://<Replacement of localminioservice address:9000>',
  'warehouse'='s3://warehouse/wh/'
);

Creating the database and Iceberg tables

create database if not exists iceberg.db_iceberg;

CREATE TABLE if not exists iceberg.db_iceberg.tb_iceberg (
    id BIGINT,
    val string,
    PRIMARY KEY (id) NOT ENFORCED
) WITH (
''='true',
'upsert-enabled'='true',
''='merge-on-read',
''='merge-on-read'
);
<Previous Next>