1. Configure the java environment of two tomcat back-end services separately
1) Prepare jdk8 compression packages
[root@tomcat jdk]# pwd /usr/local/src/jdk root@tomcat jdk]# ls jdk-8u211-linux-x64.tar.gz
2) Unzip the jdk package in the current directory and create a soft connection
[root@tomcat jdk]# tar xvf jdk-8u211-linux-x64.tar.gz [root@tomcat jdk]# ln -sv jdk1.8.0_211/ jdk
3) Configure java environment variables and take effect
[root@tomcat ~]# vim /etc/profile
......
export JAVA_HOME=/usr/local/src/jdk/jdk
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib/:$JRE_HOME/lib
export TOMCAT_HOME=/usr/local/src/tomcat/tomcat
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$TOMCAT_HOME/bin
[root@tomcat ~]# source /etc/profile
4) Testing the java environment
[root@tomcat ~]# echo $JAVA_HOME /usr/local/src/jdk/jdk [root@tomcat ~]# echo $CLASSPATH /usr/local/src/jdk/jdk/lib/:/usr/local/src/jdk/jdk/jre/lib [root@tomcat ~]# echo $PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/local/src/jdk/jdk/bin:/usr/local/src/jdk/jdk/jre/bin:/usr/local/src/tomcat/tomcat/bin:/root/bin [root@tomcat ~]# java -version #View the version of java java version "1.8.0_211" Java(TM) SE Runtime Environment (build 1.8.0_211-b12) Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)
2. Install and configure tomcat services separately
1) Prepare tomcat binary compression package
[root@tomcat tomcat]# pwd /usr/local/src/tomcat [root@tomcat tomcat]# ls apache-tomcat-8.5.43.tar.gz
2) Decompress the tomcat compressed file and create a soft connection
[root@tomcat tomcat]# tar xvf apache-tomcat-8.5.43.tar.gz [root@tomcat tomcat]# ln -sv apache-tomcat-8.5.43 tomcat
3) Start tomcat service
[root@tomcat ~]# /usr/local/src/tomcat/tomcat/bin/catalina.sh start
4) View the startup port
[root@tomcat ~]# ss -tnlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:22 *:* users:(("sshd",pid=3716,fd=3))
LISTEN 0 100 127.0.0.1:25 *:* users:(("master",pid=3936,fd=13))
LISTEN 0 100 :::8009 :::* users:(("java",pid=5861,fd=54))
LISTEN 0 100 :::8080 :::* users:(("java",pid=5861,fd=49))
LISTEN 0 128 :::22 :::* users:(("sshd",pid=3716,fd=4))
LISTEN 0 100 ::1:25 :::* users:(("master",pid=3936,fd=14))
LISTEN 0 1 ::ffff:127.0.0.1:8005 :::* users:(("java",pid=5861,fd=75))5) Browser Access Test "Main tomcat Service"
6) Browser Access Test "Standby tomcat-1 Service"
3. Configure two keepalived+haproxy high availability separate scheduling services
1) Install high availability service keepalived
[root@keepalive_haproxy ~]# yum install keepalive -y
2) Modify the keepalived configuration file
[root@keepalive_haproxy ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id haproxy #The routing id in the backup service is set to "haproxy-1", which is not the same.
vrrp_skip_check_adv_addr
# vrrp_strict #Disable vrrp, otherwise only support multicast and not unicast mode
vrrp_iptables #Open the rule of not automatically adding firewall to avoid unavailability of access to this host
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER #Set it as main service, BACKUP in backup service, backup service
interface eth0 #Binding network card
virtual_router_id 51 #Instance routing id number, which can be the same primary and standby services
priority 100 #Priority, backup service priority must be less than 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.10.23/ dev eth0 label eth0:0 #Bind virtual vip to the local eth0 network card and name it eth0:0
}
unicast_src_ip 192.168.1.10 #Unicast source address ip, set source IP to 192.168.1.11 in backup service
unicast_peer{
192.168.1.11 #Unicast destination address ip, set target IP to 192.168.1.10 in backup service
}
}3) Start the keepalived service separately
Main keepalivd:
[root@keepalive_haproxy ~]# systemctl start keepalived
root@keepalive_haproxy ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor preset: disabled)
Active: active (running) since Mon 2019-08-05 18:10:00 CST; 21s ago
Process: 4313 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 4314 (keepalived)
[root@keepalive_haproxy ~]# ip a
......
eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:36:53:00 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.10/24 brd 192.168.1.255 scope global eth0
valid_lft forever preferred_lft forever
inet 192.168.10.23/0 scope global eth0:0 #Binding virtual vip
valid_lft forever preferred_lft forever
......Keep alivd:
[root@keepalive_haproxy ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor preset: disabled)
Active: active (running) since Mon 2019-08-05 17:32:01 CST; 40min ago
Process: 3712 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 3853 (keepalived)
[root@keepalive_haproxy ~]# ip a #No virtual vip is seen. When the primary service is deactivated, the virtual vip will automatically drift to the host.
......
eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:c4:e2:07 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.11/24 brd 192.168.1.255 scope global eth0
......4) Configure two dispatching service kernel parameters separately
[root@keepalive_haproxy ~]# vim /etc/sysctl.conf ...... net.ipv4.ip_nonlocal_bind = 1 #Open non-native ip binding to avoid haproxy being unable to bind non-native ip net.ipv4.ip_forward = 1 #Turn on routing forwarding
5) Effective Kernel Parameters
[root@keepalive_haproxy ~]# sysctl -p net.ipv4.ip_nonlocal_bind = 1 net.ipv4.ip_forward = 1
6) Compile and install haproxy separately. The following is the path after compiling and installing
[root@keepalive_haproxy haproxy]# pwd /usr/local/src/haproxy [root@keepalive_haproxy haproxy]# ls doc sbin share
7) To modify the configuration file again, the two service profiles must remain the same
[root@keepalive_haproxy ~]# vim /etc/haproxy/haproxy.cfg Global maxconn 100000 #Maximum number of concurrent connections per process chroot /usr/local/src/haproxy #Lock the running directory #stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin stats socket /usr/local/src/haproxy/haproxy.sock mode 600 level admin #Custom sock //File path, under which the haproxy startup user must have permission to create the haproxy.sock file, otherwise the service cannot //Start, this sock file to provide manual offline back-end service function, can also be commented out not to create sock files uid 88 #User Identity Executing haproxy gid 88 #Subordinate group daemon nbproc 2 #Number of threads opened cpu-map 1 0 #Core 0 bound to cup cpu-map 2 1 #Binding to Core 1 of cup pidfile /run/haproxy.pid #pid file path log 127.0.0.1 local3 info #Define global syslog defaults #Default settings for front-end, back-end and listen option http-keep-alive option forwardfor #ip transmission maxconn 100000 mode http timeout connect 300000ms timeout client 300000ms timeout server 300000ms listen stats #Open the listen status page mode http #http protocol bind 0.0.0.0:9999 #Ports bound by status page access stats enable #Open status page log global #Global Log stats uri /haproxy-status #State also path stats auth admin:123456 #Username and password for status page login listen web_port #Monitoring services bind 192.168.10.23:80 #Binded virtual vip and ports, which are automatically scheduled to back-end services when accessed by an external network mode http #http protocol balance roundrobin #roundrobin dynamic polling log global #Global Log server 192.168.1.20 192.168.1.20:8080 check inter 3000 fall 2 rise 5 #Scheduled back-end services server 192.168.1.21 192.168.1.21:8080 check inter 3000 fall 2 rise 5 #Scheduled back-end services
8) Create haproxy startup users
[root@keepalive_haproxy haproxy]# useradd -r -s /sbin/nologin haproxy -u 88
9) Create haproxy startup scripts separately
[root@keepalive_haproxy haproxy]# vim /usr/lib/systemd/system/haproxy.service [Unit] Description=HAproxy Load Balancer After=syslog.target network.target [Service] ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid ExecStop=/bin/kill -USR2 $MAINPID [Install] WantedBy=multi-user.target
10) Start haproxy service separately
[root@keepalive_haproxy haproxy]# systemctl start haproxy
root@keepalive_haproxy haproxy]# systemctl status haproxy
● haproxy.service - HAproxy Load Balancer
Loaded: loaded (/usr/lib/systemd/system/haproxy.service; enabled; vendor preset: disabled)
Active: active (running) since Mon 2019-08-05 17:31:48 CST; 1h 25min ago
Process: 3716 ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q (code=exited, status=0/SUCCESS)
Main PID: 3769 (haproxy)
[root@keepalive_haproxy haproxy]# ss -tnlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:9999 *:* users:(("haproxy",pid=3828,fd=5),("haproxy",pid=3827,fd=5))
LISTEN 0 128 192.168.10.23:80 *:* users:(("haproxy",pid=3828,fd=7),("haproxy",pid=3827,fd=7))
......11)haproxy status page access
12) Browsers access scheduling services and successfully schedule back-end services
4. Create the execution script of Jenkins to implement automatic testing, deployment and rollback code through Jenkins'option parameters (pre-built services such as jenkins, gitlab, sonaqube, etc., in which Jenkins installs scanner scanner)
1) Customize the creation of the specified jenkins service working directory
[root@jenkins]# mkdir /data/jenkins/worker -pv
2) save path of jenkins server script
[root@jenkins jenkins]# pwd /data/jenkins
3) jenkins server edit script
[root@jenkins jenkins]# vim project.sh
#!/bin/bash
#jenkins parameter options
time=`date +%Y-%m-%d_%H-%M-%S`
method=$1
group=$2
branch=$3
#Back-end tomcat service ip address group
ip_value(){
if [[ $group == "group1" ]];then
ip_list="192.168.1.20"
echo ${ip_list}
elif [[ $group == "group2" ]];then
ip_list="192.168.1.21"
echo ${ip_list}
ssh root@192.168.1.10 "echo "enable server web_port/192.168.1.20" | socat stdio /usr/local/src/haproxy/haproxy.sock"
ssh root@192.168.1.11 "echo "enable server web_port/192.168.1.20" | socat stdio /usr/local/src/haproxy/haproxy.sock"
elif [[ $group == "group3" ]];then
ip_list="192.168.1.20 192.168.1.21"
echo ${ip_list}
fi
}
#The code is first deployed to the Jenkins server
code_deploy(){
cd /data/jenkins/worker
rm -rf ./*
git clone -b $branch git@192.168.1.30:jie/web-page.git
}
#Code testing
code_test(){
cd /data/jenkins/worker/web-page
cat > sonar-project.properties <<eof
sonar.projectKey=one123456
sonar.projectName=code-test
sonar.projectVersion=1.0
sonar.sources=./
sonar.language=php
sonar.sourceEncoding=UTF-8
eof
/data/scanner/sonar-scanner/bin/sonar-scanner
}
#Code Compression
code_compress(){
cd /data/jenkins/worker/
rm -f web-page/sonar-project.properties
tar czvf code-tar.gz web-page
}
#Scheduler strips back-end services
haproxy_down(){
for ip in ${ip_list};do
echo $ip
ssh root@192.168.1.10 "echo "disable server web_port/${ip}"|socat stdio /usr/local/src/haproxy/haproxy.sock"
ssh root@192.168.1.11 "echo "disable server web_port/${ip}"|socat stdio /usr/local/src/haproxy/haproxy.sock"
done
}
#Back-end service offline
backend_stop(){
for ip in ${ip_list};do
echo $ip
ssh root@$ip "/usr/local/src/tomcat/tomcat/bin/catalina.sh stop"
done
}
#Deploy code to back-end service sites
scp_backend(){
for ip in ${ip_list};do
echo $ip
scp /data/jenkins/worker/code-tar.gz root@${ip}:/usr/local/src/tomcat/tomcat/web-code/${time}-code-tar.gz
ssh root@${ip} "tar xvf /usr/local/src/tomcat/tomcat/web-code/${time}-code-tar.gz -C /usr/local/src/tomcat/tomcat/webapps"
done
}
#Start back-end services
backend_start(){
for ip in ${ip_list};do
echo $ip
ssh root@$ip "/usr/local/src/tomcat/tomcat/bin/catalina.sh start"
sleep 6
done
}
#Test access back-end services
backend_test(){
for ip in ${ip_list};do
echo $ip
status_code=`curl -I -s -m 6 -o /dev/null -w %{http_code} http://${ip}:8080`
if [ ${status_code} -eq 200 ];then
echo "Access test successful, back-end code deployment successful"
if [[ $ip == "192.168.1.21" ]];then
ssh root@192.168.1.10 "echo "enable server web_port/${ip}" | socat stdio /usr/local/src/haproxy/haproxy.sock"
ssh root@192.168.1.11 "echo "enable server web_port/${ip}" | socat stdio /usr/local/src/haproxy/haproxy.sock"
fi
else
echo "Access test failed, redeploy code to back-end service"
fi
done
}
#Code rollback
code_rollback(){
for ip in ${ip_list};do
echo $ip
last_version=`ssh root@${ip} "ls -l -t /usr/local/src/tomcat/tomcat/web-code/" | awk 'NR==3{print $NF}'`
ssh root@${ip} " tar xvf /usr/local/src/tomcat/tomcat/web-code/$last_version -C /usr/local/src/tomcat/tomcat/webapps"
done
echo "tomcat Code rollback is successful, go back to the previous version, next step for access testing"
}
#Main menu commands
main(){
case $1 in
deploy)
ip_value;
code_deploy;
code_test;
code_compress;
haproxy_down;
backend_stop;
scp_backend;
backend_start;
backend_test;
;;
rollback)
ip_value;
haproxy_down;
backend_stop;
code_rollback;
backend_start;
backend_test;
;;
esac
}
main $1 $2 $34) Create a code compression file save path in each back end
Main tomcat:
[root@tomcat tomcat]# mkdir web-code [root@tomcat tomcat]# pwd /usr/local/src/tomcat/tomcat
Prepare tomcat-1:
[root@tomcat-1 tomcat]# mkdir web-code [root@tomcat-1 tomcat]# pwd /usr/local/src/tomcat/tomcat
5) Set up the jenkins service to login the services without secret key.
[root@jenkins jenkins]# ssh-copy-id 192.168.1.10 [root@jenkins jenkins]# ssh-copy-id 192.168.1.11 [root@jenkins jenkins]# ssh-copy-id 192.168.1.20 [root@jenkins jenkins]# ssh-copy-id 192.168.1.21
5. Clone and push code from gitlab server
1) Clone the specified develop ment branch code
root@ubuntu1804:~# git clone -b develop http://192.168.1.30/jie/web-page.git Cloning into 'web-page'... Username for 'http://192.168.1.30': jie Password for 'http://jie@192.168.1.30': remote: Enumerating objects: 39, done. remote: Counting objects: 100% (39/39), done. remote: Compressing objects: 100% (22/22), done. remote: Total 39 (delta 4), reused 27 (delta 4) Unpacking objects: 100% (39/39), done.
2) View the code file contained in the clone
root@ubuntu1804:~# ls web-page/ index.html Math.php
3) Modification of Generation Documents
root@ubuntu1804:~/web-page# cat index.html <h1>welcome to tomcat page</h1> <h3>simple-version v1</h3>
4) Push v1 code to gitlab code base
root@ubuntu1804:~/web-page# git add ./*
root@ubuntu1804:~/web-page# git commit -m 'v1'
[develop d0dd713] v1
1 file changed, 2 insertions(+), 2 deletions(-)
root@ubuntu1804:~/web-page# git push
Username for 'http://192.168.1.30': jie
Password for 'http://jie@192.168.1.30':
Counting objects: 3, done.
Delta compression using up to 4 threads.
Compressing objects: 100% (3/3), done.
Writing objects: 100% (3/3), 316 bytes | 316.00 KiB/s, done.
Total 3 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for develop, visit:
remote: http://192.168.1.30/jie/web-page/merge_requests/new?merge_request%5Bsource_branch%5D=develop
remote:
To http://192.168.1.30/jie/web-page.git
c10f5bf..d0dd713 develop -> develop6. Configuration File Modification and Option Parameter Construction of Jenkins
1) Create a project code-test
2) Configure the configure file for this project, Add option parameters, character parameters and correspond to the options in the script file
3) Configure the shell script command of jenkins, which implements code testing, deployment and rollback
4) Save the above configuration and deploy the first set of back-end service master tomcat
5) Console Output Information
6) Direct browser access to master tomcat service to verify successful deployment
7) Redeploy the second set of back-end services for tomcat-1
8) Console output deployment success information
9) View the relevant code files of the back-end service deployment separately to determine whether the code files are deployed to the back-end service or not.
The main tomcat server:
[root@tomcat tomcat]# pwd/usr/local/src/tomcat/tomcat [root@tomcat tomcat]# ll web-code/ total 16 -rw-r--r-- 1 root root 14910 Aug 4 18:23 2019-08-04_18-23-01-code-tar.gz [root@tomcat webapps]# pwd /usr/local/src/tomcat/tomcat/webapps [root@tomcat webapps]# cat web-page/index.html <h1>welcome to tomcat page</h1> <h3>simple-version v1</h3>
Prepare tomcat-1 server:
[root@tomcat-1 tomcat]# pwd /usr/local/src/tomcat/tomcat [root@tomcat-1 tomcat]# ll web-code/ total 16 -rw-r--r-- 1 root root 14910 Aug 4 18:23 2019-08-04_18-23-01-code-tar.gz [root@tomcat-1 webapps]# pwd /usr/local/src/tomcat/tomcat/webapps [root@tomcat-1 webapps]# cat web-page/index.html <h1>welcome to tomcat page</h1> <h3>simple-version v1</h3>
10) Direct browser access to standby Tomcat 1 service verifies that the deployment is successful, and the standby tomcat-1 code as shown below is also deployed successfully
11) Finally, through the browser haproxy scheduler, successfully dispatched to the back-end service tomcat
12) Code test results
7. Upgrade the back-end service code to the new version of v2
1) Update code on gitlab server
root@ubuntu1804:~/web-page# cat index.html <h1>welcome to tomcat page</h1> <h3>enhanced-version v2--- Handled bugs on the old v1</h3>
2) Push the new version of v2 code to the gitlab code base again on the gitlab server
root@ubuntu1804:~/web-page# git add ./*
root@ubuntu1804:~/web-page# git commit -m 'v2'
[develop 2512294] v2
1 file changed, 1 insertion(+), 1 deletion(-)
root@ubuntu1804:~/web-page# git push
Username for 'http://192.168.1.30': jie
Password for 'http://jie@192.168.1.30':
Counting objects: 3, done.
Delta compression using up to 4 threads.
Compressing objects: 100% (3/3), done.
Writing objects: 100% (3/3), 344 bytes | 344.00 KiB/s, done.
Total 3 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for develop, visit:
remote: http://192.168.1.30/jie/web-page/merge_requests/new?merge_request%5Bsource_branch%5D=develop
remote:
To http://192.168.1.30/jie/web-page.git
d0dd713..2512294 develop -> develop3) Build parameter group 3, upgrade all back-end services to v2 version
4) View the results of console execution and display successful deployment
5) View the updated code files at the back-end servers, check whether the code is updated, and browser test access
tomcat server:
[root@tomcat webapps]# cat web-page/index.html <h1>welcome to tomcat page</h1> <h3>enhanced-version v2--- Handled bugs on the old v1</h3>
tomcat-1 server:
[root@tomcat-1 webapps]# cat web-page/index.html <h1>welcome to tomcat page</h1> <h3>enhanced-version v2--- Handled bugs on the old v1</h3>
6) Access haproxy Scheduler service through browser to check whether the updated code is successfully scheduled to back-end service
7) Code test results
8. Code rollback to the old version (if v2 version is unstable, it is necessary to roll back back the back-end service code to the old version v1 to avoid affecting business)
1) Build Selective rollback rollback, all backend rollbacks in group 3
2) View console input code deployment output information
3) Check the back-end service code files, check whether the code rolls back to the old version of v1, and the browser tests each server.
The main tomcat server:
[root@tomcat webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>simple-version v1</h3>
Prepare tomcat-1 server:
[root@tomcat-1 webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>simple-version v1</h3>
4) Access haproxy scheduler service, code rollback successful
