Location>code7788 >text

Docker continuous integration deployment + ELK log related, etc. Perfect practice

Popularity：850 ℃/2025-02-18 14:40:50
docker(ubuntu)

Uninstall docker

# Uninstall (if any)
 for pkg in docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove $pkg; done

Install docker

# Update and install the tools
 sudo apt-get update
 sudo apt-get install ca-certificates curl

 # Add Alibaba Cloud GPG Key
 sudo curl -fsSL /docker-ce/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/

 # Configure the docker software source information to Alibaba Cloud
 sudo echo "deb [arch=amd64 signed-by=/usr/share/keyrings/] /docker-ce/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt// > /dev/null

 # Install docker
 sudo apt-get update
 sudo apt-get install docker-ce docker-ce-cli docker-buildx-plugin docker-compose-plugin

 # Start and test
 sudo systemctl start docker
 sudo docker run hello-world

 # Verify (This appears, indicating that the installation is successful)
 Hello from Docker!
 This message shows that your installation appears to be working correctly.
 ...

Change docker domestic image

# Alibaba Cloud mirror acceleration cannot be used, try using the following
 echo '{"registry-mirrors": ["https://docker."]}' | sudo tee /etc/docker/ > /dev/null
 systemctl daemon-reload
 systemctl restart docker

Change docker storage location

# Change the docker storage location, if you want to exist in the /data/docker directory
 cd /data
 mkdir docker
 # Modify /etc/docker/, add "data-root": "/data/docker" configuration to the root object

Install docker-compose

# Download docker-compose
 # The version needs to be replaced with the latest one, view from the link below
 # /docker/compose/releases
 curl -L "/docker/compose/releases/download/v2.32.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

 # Increase execution permissions
 chmod +x /usr/local/bin/docker-compose

jenkins

Pull the mirror

docker pull jenkins/jenkins

Create a mount directory

mkdir -p /data/jenkins/{jenkins_home,}

Create a startup Jenkins container

docker run -d \
  --name jenkins \
  --user $(id -u):$(id -g) \
  -p 9080:8080 \
  -p 50000:50000 \
  -v /data/jenkins/jenkins_home:/var/jenkins_home \
  -v /data/jenkins/:/var/run/ \
  -v $(which docker):/usr/bin/docker \
  --restart=always \
  jenkins/jenkins
Detailed explanation:
# Use Docker to run a new container
 docker run -d \
   # Assign a name to the container
   --name jenkins \
   # Run the process in the container as the current user, $(id -u) gets the user ID of the current user, $(id -g) gets the group ID of the current user
   --user $(id -u):$(id -g) \
   # Map the host's port 9010 to the container's port 8080 to access Jenkins' web interface
   -p 9080:8080 \
   # Map the host's port 9011 to the container's port 50000 for JNLP proxy connections
   -p 50000:50000 \
   # Mount the /data/jenkins/jenkins_home directory on the host to the /var/jenkins_home directory of the container to persist Jenkins data
   -v /data/jenkins/jenkins_home:/var/jenkins_home \
   # Mount the Docker socket file on the host to the container so that Jenkins in the container can use the host's Docker daemon
   -v /data/jenkins/:/var/run/ \
   # Mount the Docker command on the host machine into the container so that Jenkins in the container can execute Docker commands
   -v $(which docker):/usr/bin/docker \
   # Specify the image to run. Here is the Jenkins image provided by Jenkins
   jenkins/jenkins

View jenkins unlock password

docker logs  -f jenkins
As shown below
 *********************************************************  ************
 *********************************************************  ************
 *********************************************************  ************

 Jenkins initial setup is required. An admin user has been created and a password generated.
 Please use the following password to proceed to installation:

 6f1c8e1131974079ace00e89507c6a16

 This may also be found at: /var/jenkins_home/secrets/initialAdminPassword

 *********************************************************  ************
 *********************************************************  ************
 *********************************************************  ************

Set password

admin admin123

Jenkins dotnet command cannot be executed

docker exec -it jenkins bash

#dotnet
apt-get install wget

wget /config/ubuntu/20.04/ -O 

dpkg -i 

apt-get update

apt-get install dotnet-sdk-6.0

Add a custom Nuget source

 
docker exec -it jenkins bash

 cd ~/.nuget/NuGet
 # Add the first source#
 sed -i '/<\/packageSources>/i \ <add key="ZbirdNuGet" value="/nuget" />'

 # Add a second source
 sed -i '/<\/packageSources>/i \ <add key="ZbirdSandboxNuGet" value="/nuget" />'

Pack existing containers into mirror images

# docker commit -m "Submit information" -a "Author" <Container ID or Name> mynewimage:latest
 docker commit -m 'configured jenkins' -a 'TE' jenkins myjenkins:1.0

Configure NodeJs

System Management => Plugin Management => Install NodeJS
 System Management => Global Tool Configuration => NodeJS Installation => Check Automatic Installation Select the required NodeJS version
 Configure pipeline => Environment => Provide Node and npm bin/ folder for PATH (Provide Node & npm bin/ folder to PATH) => Select the nodejs name you configured in the global tool

clickhouse

Pull the mirror

# Pull the mirror
 docker pull yandex/clickhouse-server

Configuration File

# Start the temporary container and copy the configuration files in the container
 docker run -d --rm --name=temp yandex/clickhouse-server
 # -d Backend Run
 # --rm Starts the temporary container. When the container is stopped, the container will be automatically deleted.
 # --name Container name

 # Create related directories on the storage disk
 mkdir -p /data/clickhouse/conf /data/clickhouse/data/clickhouse/data/clickhouse/log

 # Copy the configuration file in the container to the storage disk
 docker cp temp:/etc/clickhouse-server/ /data/clickhouse/conf/
 docker cp temp:/etc/clickhouse-server/ /data/clickhouse/conf/

 # Generate SHA256 password
 PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
 # Return result
 XwCoKBgV #Password clear text (to record it, that is, access the login password of clickhourse)
 2c297a5ee6d922c0472dee50d3067ea1ce99dd54e765247e287f9ca262525a63 #Secretary
 # Modify /data/clickhouse/conf/
 # Change the <password> node to the following
 # <password_sha256_hex>2c297a5ee6d922c0472dee50d3067ea1ce99dd54e765247e287f9ca262525a63</password_sha256_hex>

Start the container

# Start the container
 docker run -d --name clickhouse-server -p 8123:8123 \
 -p 9009:9009 \
 -p 9090:9000 \
 --ulimit nofile=262144:262144 \
 --volume=/data/clickhouse/data:/var/lib/clickhouse \
 --volume=/data/clickhouse/log:/var/log/clickhouse-server \
 --volume=/data/clickhouse/conf/:/etc/clickhouse-server/ \
 --volume=/data/clickhouse/conf/:/etc/clickhouse-server/ \
 --restart=always \
 yandex/clickhouse-server

 # Default login name: default
 # Password: "Password clear text" returned when SHA256 is generated

apollo

Deploy mysql

# Pull mysql image
 docker pull mysql:5.7.44

 # Create mysql mount directory
 mkdir -p /data/mysql/{log,data,}
 # Create mysql configuration file
 echo "" > /data/mysql/

 # Start mysql container
 docker run -d --restart=always --name mysql \
 -v /data/mysql/data:/var/lib/mysql \
 -v /data/mysql/log:/var/log/mysql \
 -v /data/mysql/:/etc/mysql/ \
 -v /data/mysql/:/etc/ \
 -p 3306:3306 \
 -e TZ=Asia/Shanghai \
 -e MYSQL_ROOT_PASSWORD=NCwkaR14 \
 mysql:5.7.44

Initialize the database

  1. Deploy local apollo using docker-compose - Gao Hongshun - Blog Park
  2. Finally, create a multi-environment database, one for each environment of ApolloConfigDB

Pull the apollo image

docker pull apolloconfig/apollo-portal
docker pull apolloconfig/apollo-configservice
docker pull apolloconfig/apollo-adminservice

create

# Create a directory
 mkdir -p /etc/apollo

 # Create a docker-compose file, copy the ymal below and save it
 vim /etc/apollo/
version: '4'
 services:
   # DEV
   apollo-configservice-dev:
     image: apolloconfig/apollo-configservice
     environment:
       # jdbc mysql address
       SPRING_DATASOURCE_URL: jdbc:mysql://192.168.0.16:3306/ApolloConfigDB_DEV?useSSL=false&characterEncoding=utf8
       SPRING_DATASOURCE_USERNAME: root
       SPRING_DATASOURCE_PASSWORD: NCwkaR14 # Password needs to be replaced
       SERVER_PORT: 8080
     network_mode: host

   apollo-adminservice-dev:
     image: apolloconfig/apollo-adminservice
     depends_on:
       - apollo-configservice-dev
     environment:
       # jdbc mysql address
       SPRING_DATASOURCE_URL: jdbc:mysql://192.168.0.16:3306/ApolloConfigDB_DEV?useSSL=false&characterEncoding=utf8
       SPRING_DATASOURCE_USERNAME: root
       SPRING_DATASOURCE_PASSWORD: NCwkaR14 # Password needs to be replaced
       SERVER_PORT: 8090
     network_mode: host

   apollo-configservice-pro:
     image: apolloconfig/apollo-configservice
     environment:
       # jdbc mysql address
       SPRING_DATASOURCE_URL: jdbc:mysql://192.168.0.16:3306/ApolloConfigDB_PRO?useSSL=false&characterEncoding=utf8
       SPRING_DATASOURCE_USERNAME: root
       SPRING_DATASOURCE_PASSWORD: NCwkaR14 # Password needs to be replaced
       SERVER_PORT: 8081
     network_mode: host

   apollo-adminservice-pro:
     image: apolloconfig/apollo-adminservice
     depends_on:
       - apollo-configservice-pro
     environment:
       # jdbc mysql address
       SPRING_DATASOURCE_URL: jdbc:mysql://192.168.0.16:3306/ApolloConfigDB_PRO?useSSL=false&characterEncoding=utf8
       SPRING_DATASOURCE_USERNAME: root
       SPRING_DATASOURCE_PASSWORD: NCwkaR14 # Password needs to be replaced
       SERVER_PORT: 8091
     network_mode: host

   apollo-portal:
     image: apolloconfig/apollo-portal
     depends_on:
       - apollo-adminservice-dev
       - apollo-adminservice-pro
     environment:
       # jdbc mysql address
       SPRING_DATASOURCE_URL: jdbc:mysql://192.168.0.16:3306/ApolloPortalDB?useSSL=false&characterEncoding=utf8
       SPRING_DATASOURCE_USERNAME: root
       SPRING_DATASOURCE_PASSWORD: NCwkaR14 # Password needs to be replaced
       SERVER_PORT: 8070
     network_mode: host

Execute the docker command

# Install yaml
 docker-compose -f /etc/apollo/ up -d
 # Default account password: Username: apollo Password: admin

 # Uninstall yaml
 docker-compose -f /etc/apollo/ down

 # Restart yaml
 docker-compose -f /etc/apollo/ restart

docker private warehouse

Pull the mirror

# Pull the registry image
 docker pull registry

Modify the daemon file of docker

"insecure-registries": ["192.168.0.16:5000"]
  1. Add "insecure-registries" node:
    1. Just use IP+port

Restart docker

# Reload the file
 systemctl daemon-reload
 # Restart docker
 systemctl restart docker

Registry configuration file

There are two ways, just select one of them, either way, create a directory first
# Create a configuration file storage directory
 mkdir -p /data/docker/registry

Method 1: Copy from inside the container

# Start a container
 docker run -d -p 5000:5000 --name registry2 -v /data/etc/registry:/var/lib/ registry registry
 # Copy the configuration file in the container
 docker cp registry2:/etc/docker/registry/ /data/docker/registry/
 # Delete the container
 docker rm registry2
 # Modify the configuration file and add Access-Control-Allow-Origin and Access-Control-Allow-Methods nodes
 http:
   addr: :5000
   headers:
     X-Content-Type-Options: [nosniff]
     # Add this line
     Access-Control-Allow-Origin: ['*']
     # Add this line
     Access-Control-Allow-Methods: ['*']

Method 2: Use existing files

  1. The file name is named:
  2. Copy the following into the configuration file
# Named
 version: 0.1
 log:
   fields:
     service: registry
 Storage:
   cache:
     blobdescriptor: inmemory
   filesystem:
     rootdirectory: /var/lib/registry
 http:
   addr: :5000
   headers:
     X-Content-Type-Options: [nosniff]
     Access-Control-Allow-Origin: ['*']
     Access-Control-Allow-Methods: ['*']
 health:
   Storagedriver:
     enabled: true
     interval: 10s
     threshold: 3

Start the mirror warehouse container

# Start the container
 docker run -d -p 5000:5000 --name registry \
 -v /data/docker/registry:/var/lib/registry \
 -v /data/docker/registry/:/etc/docker/registry/ \
 --restart=always \
 Registry

Test whether the container is started successfully

# Test whether the deployment is successful
 curl -XGET http://192.168.0.16:5000/v2/_catalog

 # If successful, it will return:
 # {"repositories":[]}

Test push and pull mirror

# Test push mirror
 docker pull nginx

 # Tag
 docker tag nginx:latest 192.168.0.16:5000/nginx

 # Upload
 docker push 192.168.0.16:5000/nginx

 # Verify whether the image is uploaded successfully
 curl -XGET http://192.168.0.16:5000/v2/_catalog
 # If the push is successful, it will return:
 # {"repositories":["nginx"]}

 # Pull the mirror
 # Delete existing images
 docker rmi 192.168.0.16:5000/nginx:latest

 # Pull the mirror
 docker pull 192.168.0.16:5000/nginx

es+kibana

version: '3'
 services:
   elasticsearch:
     image: elasticsearch:8.16.2
     restart: unless-stopped
     container_name: elasticsearch
     environment:
       - "=single-node"
       - "=docker-cluster"
       - "=0.0.0.0"
       - "=true"
       - "-headers=Authorization,X-Requested-With,Content-Length,Content-Type"
       - "=false"
       - "ELASTIC_PASSWORD=123456"
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
     Volumes:
       - /data/elasticsearch/data:/usr/share/elasticsearch/data
       - /data/elasticsearch/logs:/usr/share/elasticsearch/logs
       - /data/elasticsearch/plugins:/usr/share/elasticsearch/plugins
         #- /data/elasticsearch/config:/usr/share/elasticsearch/config # There is no permission in the container to obtain permissions for this folder
     Ports:
       - "9200:9200"
       - "9300:9300"
     networks:
       - elastic_net

   Kibana:
     image: kibana:8.16.2
     container_name: kibana
     restart: unless-stopped
     depends_on:
       - elasticsearch
     environment:
       - "ELASTICSEARCH_HOSTS=http://elasticsearch:9200"
       - "ELASTICSEARCH_USERNAME=elastic"
       - "ELASTICSEARCH_PASSWORD=123456"
       - ":true"
     Ports:
       - "5601:5601"
     networks:
       - elastic_net

 networks:
   elastic_net:
     driver: bridge

Due to container permission issues, some files need to be first added

#- /data/elasticsearch/config:/usr/share/elasticsearch/config # There is no permission in the container to obtain permissions for this folder
  1. Comment this line first and run the container
  2. Copy the configuration file to the local
  3. Create a user
  4. Uninstall the container
  5. Reinstall

Copy the file

docker cp elasticsearch:/usr/share/elasticsearch/data /opt/elasticsearch/
docker cp elasticsearch:/usr/share/elasticsearch/plugins /opt/elasticsearch/
docker cp elasticsearch:/usr/share/elasticsearch/config /opt/elasticsearch/

Create a user

Create a new account
 elasticsearch-users useradd kibana

 Authorize the account
 elasticsearch-users roles -a superuser username
 elasticsearch-users roles -a kibana_system username
 

nuget

redis

Pull the mirror

docker pull redis

Create a mapping directory

mkdir -p /data/redis/{conf,data,log}

Start redis

docker run -d \
  --name redis \
  -p 6379:6379 \
  -v /data/redis/conf/:/etc/redis/ \
  -v /data/redis/data:/data \
  -v /data/redis/log:/var/log/redis \
  --restart=always \
  redis redis-server --requirepass '!QA2ws3ed'

nginx

Pull the mirror

docker pull nginx

Create a mapped folder

mkdir -p /data/nginx/{conf,html,log}

Create a container and copy the configuration file

#Generate container
 docker run --name nginx -p 9001:80 -d nginx
 #Copy the container file to the host machine
 docker cp nginx:/etc/nginx/ /data/nginx/conf/
 #Copy the contents in the container folder to the host machine
 docker cp nginx:/etc/nginx/ /data/nginx/conf/
 #Copy the html folder in the container to the host machine
 docker cp nginx:/usr/share/nginx/html /data/nginx/

Uninstall the mirror

docker stop nginx
docker rm nginx

Install the mirror

docker run -p 80:80 --name nginx \
-v /data/nginx/conf/:/etc/nginx/  \
-v /data/nginx/conf/:/etc/nginx/  \
-v /data/nginx/log:/var/log/nginx  \
-v /data/nginx/html:/usr/shard/nginx/html -d \
--restart=always \
nginx

Created under /data/nginx/conf/

# Define an upstream server group named jenkins_upstream
 server {
     listen 80;

     server_name ;

     # Access log configuration
     access_log /var/log/nginx/;
     error_log /var/log/nginx/;
    
     proxy_set_header Host $host;
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     proxy_set_header X-Forwarded-Proto $scheme;

     # SSL configuration (if required)
     # listen 443 ssl;
     # ssl_certificate /etc/nginx/ssl/;
     # ssl_certificate_key /etc/nginx/ssl/;
     #ssl_protocols TLSv1.2 TLSv1.3;
     # ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256...';
     # ssl_prefer_server_ciphers on;

     # HSTS configuration (if required)
     # add_header Strict-Transport-Security "max-age=31536000" always;

     # Set the client request size limit
     client_max_body_size 100m;
     client_body_buffer_size 100k;
    
      # Set the timeout time for proxy connection
     proxy_connect_timeout 60s;
     proxy_send_timeout 60s;
     proxy_read_timeout 60s;

     # Set the root directory (if needed)
     # root /usr/share/nginx/html;

     location / {
         proxy_pass http://192.168.0.16:9080;
         add_header Strict-Transport-Security "max-age=31536000";
     }

     # If needed, you can add more location blocks to handle specific paths or file types

     # Error page configuration
     error_page 404 /;
     location = / {
         root /usr/share/nginx/html;
         internal;
     }

     error_page 500 502 503 504 /;
     location = / {
         root /usr/share/nginx/html;
     }
 }

Restart nginx

docker restart  nginx
 
<Previous Next>