1. HAProxy
1.1 introduction to haproxy
HAProxy is a solution that supports virtual hosts by providing high availability, load balancing and proxy based on TCP and HTTP applications. Since 2013, HAProxy has been divided into community version and enterprise version.
Enterprise website https://www.haproxy.com/
Community website https://www.haproxy.org/
1.2 version function
| function | Community Edition | Enterprise Edition |
|---|---|---|
| Advanced HTTP / TCP load balancing and persistence | √ | √ |
| Advanced health examination | √ | √ |
| Application acceleration | √ | √ |
| Advanced security features | √ | √ |
| senior management | √ | √ |
| New features of HAProxy Dev Branch | × | √ |
| 24 * 7 support services | × | √ |
| Real time dashboard | × | √ |
| VRRP and Route Health Injection HA tools | × | √ |
| ACL, mapping and TLS ticket key synchronization | × | √ |
| Advanced application based DDoS and Bot protection (automatic protection) | × | √ |
| BOT (robot) monitoring | × | √ |
| Web application firewall | × | √ |
| HTTP protocol authentication | × | √ |
| Real time cluster tracking | × | √ |
1.3 installation
1.3.1 yum
[root@haproxy-keepalived_1 ~]# yum -y install haproxy # Verify haproxy version [root@haproxy-keepalived_1 ~]# haproxy -v HA-Proxy version 1.5.18 2016/05/10 Copyright 2000-2016 Willy Tarreau <willy@haproxy.org>
1.3.2 compilation and installation
# This installation uses the latest stable version v2 2.13 official document description http://cbonte.github.io/haproxy-dconv/2.2/intro.html [root@haproxykeepalived_1 ~]# cd /usr/local/src [root@haproxykeepalived_1 src]# wget https://www.haproxy.org/download/2.2/src/haproxy-2.2.13.tar.gz [root@haproxykeepalived_1 src]# tar -zxvf haproxy-2.2.13.tar.gz [root@haproxykeepalived_1 src]# cd ./haproxy-2.2.13 # Installation dependency [root@haproxykeepalived_1 haproxy-2.2.13]# yum -y install gcc openssl-devel pcre-devel systemd-devel # Compile with reference to INSTALL [root@haproxykeepalived_1 haproxy-2.2.13]# ls BRANCHES CONTRIBUTING haproxy LICENSE README scripts tests CHANGELOG doc include MAINTAINERS reg-tests src VERDATE contrib examples INSTALL Makefile ROADMAP SUBVERS VERSION [root@haproxykeepalived_1 haproxy-2.2.13]# make -j $(nproc) TARGET=linux-glibc \ > USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 USE_PCRE=1 USE_SYSTEMD=1 Makefile:584: *** unable to automatically detect the Lua library name, you can enforce its name with LUA_LIB_NAME=<name> (where <name> can be lua5.3, lua53, lua, ...). stop it. # To solve the problem of lua version, CentOS7 automatically the lower version of lua, which does not meet the requirements of HAProxy. The minimum version of lua is 5.3 [root@haproxykeepalived_1 haproxy-2.2.13]# lua -v Lua 5.1.4 Copyright (C) 1994-2008 Lua.org, PUC-Rio [root@haproxykeepalived_1 haproxy-2.2.13]# cd .. [root@haproxykeepalived_1 src]# wget https://www.lua.org/ftp/lua-5.4.3.tar.gz [root@haproxykeepalived_1 src]# tar -zxvf lua-5.4.3.tar.gz [root@haproxykeepalived_1 src]# cd ./lua-5.4.3 [root@haproxy-keepalived_2 lua-5.4.3]# make [root@haproxy-keepalived_2 lua-5.4.3]# make install [root@haproxy-keepalived_2 lua-5.4.3]# cd ../haproxy-2.2.13 [root@haproxykeepalived_1 haproxy-2.2.13]# make -j $(nproc) TARGET=linux-glibc \ > USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 USE_PCRE=1 USE_SYSTEMD=1 [root@haproxykeepalived_1 haproxy-2.2.13]# make install [root@haproxy-keepalived_1 haproxy-2.2.13]# haproxy -v HA-Proxy version 2.2.13-5f3eb59 2021/04/02 - https://haproxy.org/ Status: long-term supported branch - will stop receiving fixes around Q2 2025. Known bugs: http://www.haproxy.org/bugs/bugs-2.2.13.html Running on: Linux 3.10.0-1160.el7.x86_64 #1 SMP Mon Oct 19 16:18:59 UTC 2020 x86_64
1.4 detailed explanation of configuration file
The configuration file of haproxy consists of two parts: global setting and proxy setting. It is divided into five sections: global, defaults, frontend, backend and listen.
# Global configuration information
global
maxconn 100000 # Maximum concurrent connections per haproxy process
log 127.0.0.1 local3 # [err warning info debug]
chroot /usr/local/haproxy # Path where chroot runs
uid 99 # Running user uid /user
gid 99 # Running user group / group
daemon # Running haproxy in background mode
nbproc 8 # Number of processes (multiple processes can be set to improve performance)
nbthread 10 # Number of threads
cpu-map 1 0 # Bind the first worker process of haproxy to CPU 0
pidfile /usr/local/haproxy/haproxy.pid # The pid storage path of haproxy. The user who starts the process must have permission to access this file
spread-checks n # The peak shifting health examination is randomly advanced or delayed by 100%. 2-5 (20% - 50%) is recommended, and 0 is the default
ulimit-n 65535 # Quantity limit of ulimit
# Global default
# These parameters can be used to configure the frontend, backend and listen components
defaults
log global
mode http # Categories processed (#Layer 7 HTTP; Layer 4 tcp)
maxconn 20480 # Maximum concurrent connections per haproxy process
option httplog # Log category http log format
option httpclose # Actively close the http channel after each request
option dontlognull # Log information of health check is not recorded
option forwardfor # If the back-end server needs to obtain the parameters that the real ip of the client needs to be configured, the client ip can be obtained from the Http Header
option redispatch # After the server corresponding to serverId hangs up, it is forced to direct to other healthy servers
option abortonclose # When the server load is very high, it will automatically end the connection that has been in the current queue for a long time
stats refresh 30 # Statistics page refresh interval
retries 3 # If the connection fails for 3 times, the service is considered unavailable, which can also be set later
balance roundrobin # The default load balancing mode is polling mode
#balance source # The default load balancing method is similar to nginx ip_hash
#balance leastconn # The default load balancing method is the minimum connection
contimeout 5000 # connection timed out
clitimeout 50000 # Client timeout
srvtimeout 50000 # server time-out
timeout check 2000 # Heartbeat detection timeout
# Monitoring page settings
listen admin_status # The combination of Frontend and Backend, the name of the monitoring group, and the custom name on demand
bind 0.0.0.0:65532 # Listening port
mode http # Layer 7 mode of http
log 127.0.0.1 local3 err # Error logging
stats refresh 5s # Automatically refresh the monitoring page every 5 seconds
stats uri /admin?stats # url of the monitoring page
stats realm jfedu\ jfedu # Prompt information of monitoring page
stats auth admin:admin # The user and password of the monitoring page are admin, and multiple user names can be set
stats hide-version # Hide the HAproxy version information on the statistics page
stats admin if TRUE # Manually enable / disable the backend server;
# Monitor the status of the haproxy backend server
listen site_status
bind 0.0.0.0:1081 # Listening port
mode http # Layer 7 mode of http
log 127.0.0.1 local3 err # [err warning info debug]
monitor-uri /site_status # The website health detection URL is used to detect whether the website managed by HAProxy can be used. It returns 200 normally and 503 abnormally
acl site_dead nbsrv(server_web) lt 2 # Define the policy when the website is down. When the number of effective machines in the specified backend hanging on the load balancing is less than 1, return true
monitor fail if site_dead # When the policy is met, 503 is returned. The online document says 500 and the actual test is 503
monitor-net 192.168.149.129/32 # The log information from 192.168.149.129 will not be recorded and forwarded;
monitor-net 192.168.149.130/32 # The log information from 192.168.149.130 will not be recorded and forwarded;
# frontend configuration
# Note that multiple ACLS can be defined in the frontend configuration for matching operations
frontend http_80_in
bind 0.0.0.0:80 # The listening port, that is, the port where haproxy provides web services, is similar to the vip port of lvs
mode http # Layer 7 mode of http
log global # Apply global log configuration
option httplog # http enabled log
option httpclose # Actively close the http channel after each request. Ha proxy does not support keep alive mode
option forwardfor #If the backend server needs to obtain the real IP of the client and needs to configure secondary parameters, the client IP can be obtained from the Http Header
# acl policy configuration
acl jfedu_web hdr_reg(host) -i ^(www1.jfedu.net|www2.jfedu.net)$
#If the requested domain name satisfies two domain names in the regular expression, return true -i to ignore case
#If the requested domain name meets the requirements of www.jfedu.com Net returns true -i is to ignore case
#acl jfedu hdr(host) -i jfedu.net
#If the requested domain name meets jfedu Net returns true -i is to ignore case
#acl file_req url_sub -i killall=
#If kill = is included in the request url, this control policy returns true, otherwise it is false
#acl dir_req url_dir -i allow
#If allow exists as part of the address path in the request url, this control policy returns true; otherwise, it returns false
#acl missing_cl hdr_cnt(Content-length) eq 0
#Returns true when the content length in the header of the request is equal to 0
# acl policy matching
# block if missing_cl
# When the content length in the header in the request is equal to 0, the request is prevented from returning 403
# block if !file_req || dir_req
# Block means to block the request and return 403 error. Currently, it means if the policy file is not met_ Req, or meet the policy dir_req, the request is blocked
use_backend server_web if jfedu_web
# When jfedu is satisfied_ Use server for web policy_ Web backend
# Setting of backend
backend server_web
mode http # Layer 7 mode of http
balance roundrobin # Load balancing, roundrobin averaging
cookie SERVERID # It is allowed to insert the serverid into the cookie. The serverid can be defined later
option httpchk GET /index.html # Heartbeat detection files
server web1 192.168.149.129:80 cookie web1 check inter 1500 rise 3 fall 3 weight 1
# As defined by the server, cookie 1 indicates that the serverid is web1, check inter 1500 indicates that the heartbeat rate is detected, rise 3 indicates that the server is correctly considered available for 3 times,
# fall 3 refers to three failures. It is considered that the server is unavailable, and weight represents the weight
server web2 192.168.149.130:80 cookie web2 check inter 1500 rise 3 fall 3 weight 2
# As defined by the server, cookie 1 indicates that the serverid is web2, check inter 1500 indicates that the heartbeat rate is detected, rise 3 indicates that the server is correctly considered available for 3 times,
# fall 3 refers to three failures. It is considered that the server is unavailable, and weight represents the weight.
1.5 configuration file example
global
log 127.0.0.1 local0 info
maxconn 4096
user nobody
group nobody
daemon
nbproc 1
pidfile /usr/local/haproxy/logs/haproxy.pid
defaults
mode http
retries 3
timeout connect 10s
timeout client 20s
timeout server 30s
timeout check 5s
frontend www
bind *:80
mode http
option httplog
option forwardfor
option httpclose
log global
#acl host_www hdr_dom(host) -i www.zb.com
#acl host_img hdr_dom(host) -i img.zb.com
#use_backend htmpool if host_www
#use_backend imgpool if host_img
default_backend htmpool
backend htmpool
mode http
option redispatch
option abortonclose
balance static-rr
cookie SESSION_COOKIE insert indirect nocache
option httpchk GET /index.jsp
server 237server 192.168.81.237:8080 cookie server1 weight 6 check inter 2000 rise 2 fall 3
server iivey234 192.168.81.234:8080 cookie server2 weight 3 check inter 2000 rise 2 fall 3
backend imgpool
mode http
option redispatch
option abortonclose
balance static-rr
cookie SERVERID
option httpchk GET /index.jsp
server host236 192.168.81.236:8080 cookie server1 weight 6 check inter 2000 rise 2 fall 3
listen admin_stats
bind 0.0.0.0:9188
mode http
log 127.0.0.1 local0 err
stats refresh 30s
stats uri /haproxy-status
stats realm welcome login\ Haproxy
stats auth admin:admin~!@
stats hide-version
stats admin if TRUE
1.6 start service & Verification
[root@haproxy-keepalived_1 ~]# haproxy -f /usr/local/src/haproxy-2.2.13/haproxy2.cfg [root@haproxy-keepalived_1 ~]# curl localhost web01 [root@haproxy-keepalived_1 ~]# curl localhost web02
2. keepalived
2.1 introduction to keepalived
Kept is the next lightweight high availability (HA) solution for Linux.
Originally designed for LVS, Keepalived is specially used to monitor the status of each service node in the cluster system. It detects the status of each service node according to the switching mechanisms of layer 3, layer 4 and layer 5 of the TCP/IP reference model. If a server node is abnormal or fails, Keepalived will detect it, And remove the failed server nodes from the cluster system. All these works are completed automatically without manual intervention. What needs to be completed manually is to repair the failed service nodes.
Later, Keepalived added the function of VRRP. The purpose of VRRP (Virtual Router Redundancy Protocol) is to solve the single point of failure of static routing. Through VRRP, the continuous and stable operation of the network can be realized. Therefore, Keepalived has the functions of server state detection and fault isolation on the one hand, and HA cluster on the other hand.
Official website: http://keepalived.org/
2.2 VRRP protocol and working principle
VRRP can virtualize two or more physical router devices into a virtual route. The virtual router provides services externally through virtual IP (one or more), while more than a dozen physical routers work together inside the virtual router, and only one physical router provides services externally at the same time, This physical routing device is called the master router (master role). Generally, the master is generated by the election algorithm. It has a virtual IP for external services and provides various network functions, such as ARP request, ICMP data forwarding, etc. in addition, other physical routers do not have an external virtual IP or provide external network functions, Only receive the VRRP status notification information from the master. These routers are collectively referred to as "BACKUP role". When the master router fails, the BACKUP router in the BACKUP role will be re elected to generate a new master router to enter the master role and continue to provide external services. The whole handover is completely transparent to users.
Each virtual router has a unique identification number called VRID. A VRID and a group of IP addresses form a virtual router. In VRRP protocol, all messages are sent through IP multicast, while in a virtual router, only the router in the master role will always send VRRP packets, The router in the BACKUP role will only accept the message information sent by the master role to monitor the operation status of the master. Generally, BACKUP preemption will not occur unless its priority is higher. When the master is unavailable, BACKUP will not receive the information sent by the master, so it is determined that the master is faulty, and then multiple bakups will vote, The BACKUP with the highest priority will be called the new master. This election role switching is very fast, thus ensuring the continuous availability of the service.
2.3 working principle of keepalvied
Keepalived works in three, four and five layers of TCP/IP reference model, namely network layer, transport layer and application layer. According to the functions that can be realized by TCP and IP parameter model, the operation mechanism of keepalived is as follows:
At the network layer: we know that these four important protocols, Internet IP protocol, Internet controllable Message Protocol ICMP, address translation protocol ARP, reverse address translation protocol RARP, In the network layer, the most common working mode of Keepalived in the network layer is to send an ICMP packet to each node in the server cluster through the ICMP Protocol (a bit similar to the function of Ping). If a node does not return a response packet, it is considered that the node has failed, and Keepalived will report that the node has failed, And eliminate the failed nodes from the server cluster.
In the transport layer: two main protocols are provided: transmission control protocol TCP and user data protocol UDP. Transmission control protocol TCP can provide reliable data output service, IP address and port, representing a connection end of TCP. To obtain TCP service, it is necessary to establish a connection between a port of the transmitter and a port of the receiver, In the transport layer, Keepalived uses the port connection and scanning technology of TCP protocol to judge whether the port of the cluster node is normal, such as port 80 of the common WEB server. Or SSH service port 22. Once Keepalived detects that these port numbers have no data response and data return at the transport layer, it considers that these ports are abnormal, and then forces the nodes corresponding to these ports to be removed from the server cluster.
In the application layer: you can run FTP, TELNET, SMTP, DNS and other different types of high-level protocols, and the operation mode of keepalived is more comprehensive and complex. Users can customize the working mode of keepalived, for example, you can run keepalived by writing programs or scripts, Keepalived will detect whether various programs or services are allowed to work normally according to the user's setting parameters. If the keepalived detection results are inconsistent with the user's setting, keepalived will eliminate the corresponding server from the server cluster.
2.4 installation
2.4.1 yum
[root@haproxy-keepalived_1 ~]# yum -y install keepalived # Verify version [root@haproxy-keepalived_1 ~]# keepalived -v Keepalived v1.3.5 (03/19,2017), git commit v1.3.5-6-g6fa32f2 Copyright(C) 2001-2017 Alexandre Cassen, <acassen@gmail.com> Build options: PIPE2 LIBNL3 RTA_ENCAP RTA_EXPIRES RTA_PREF RTA_VIA FRA_OIFNAME FRA_SUPPRESS_PREFIXLEN FRA_TUN_ID RTAX_CC_ALGO RTAX_QUICKACK LIBIPTC LIBIPSET_DYNAMIC LVS LIBIPVS_NETLINK VRRP VRRP_AUTH VRRP_VMAC SOCK_NONBLOCK SOCK_CLOEXEC FIB_ROUTING INET6_ADDR_GEN_MODE SNMP_V3_FOR_V2 SNMP SNMP_KEEPALIVED SNMP_CHECKER SNMP_RFC SNMP_RFCV2 SNMP_RFCV3 SO_MARK
2.4.2 compilation and installation
[root@haproxykeepalived_1 ~]# cd /usr/local/src [root@haproxykeepalived_1 src]# wget https://www.keepalived.org/software/keepalived-2.0.20.tar.gz [root@haproxykeepalived_1 src]# tar -zxvf keepalived-2.0.20.tar.gz [root@haproxykeepalived_1 src]# cd keepalived-2.0.20/ [root@haproxykeepalived_1 keepalived-2.0.20]# ./configure --prefix=/usr/local/keepalived ... *** WARNING - this build will not support IPVS with IPv6. Please install libnl/libnl-3 dev libraries to support IPv6 with IPVS. # In case of the above error report, the corresponding dependency of the installation [root@haproxykeepalived_1 keepalived-2.0.20]# yum install -y libnl libnl-devel [root@haproxykeepalived_1 keepalived-2.0.20]# cp keepalived/etc/init.d/keepalived /etc/init.d/ [root@haproxykeepalived_1 keepalived-2.0.20]# mkdir /etc/keepalived [root@haproxykeepalived_1 keepalived-2.0.20]# cp ./keepalived/etc/keepalived/keepalived.conf /etc/keepalived/ [root@haproxykeepalived_1 keepalived-2.0.20]# cp ./keepalived/etc/sysconfig/keepalived /etc/sysconfig/
2.5 detailed explanation of configuration file
2.5.1 composition of configuration file
GLOBAL CONFIGURATION Global definitions:Define mail configuration,route_id,vrrp to configure,Multicast address, etc VRRP CONFIGURATION VRRP instances:Define each vrrp Virtual router LVS CONFIGURATION Virtual server group Virtual server:LVS Clustered VS and RS
2.5.2 global configuration
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc #Mail alarm
}
notification_email_from Alexandre.Cassen@firewall.loc Specify sender
smtp_server 192.168.200.1 #Specify smtp server address
smtp_connect_timeout 30 appoint smtp Connection timeout
router_id LVS_DEVEL #The load balancing ID should be unique in the LAN.
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
notification_email: Specify when keepalived Which users to send mail to when a problem occurs.
notification_emai_from: When sending a message, the source address of the message.
smtp_server <DOMAIN|IP> [<PORT>]: smtp The address or domain name of the server. The default port is 25.For example: smtp_server smtp.felix.com 25
smtp_helo_name <HOST_NAME>: Specify in HELO The name used in the message. The default is the local host name.
smtp_connect_timeout: appoint smtp Timeout for server connection. The unit is seconds.
router_id: Specify the that identifies the machine route_id. For example: route_id LVS_01
vrrp_mcast_group4 224.0.0.18: Specify send VRRP Multicast message usage IPV4 Multicast address. The default is 224.0.0.18
vrrp_mcast_group6 ff02::12 Specify send VRRP Used for multicast messages IPV6 Multicast address. Default is ff02::12
default_interface eth0: Set the default binding port of static address. Default is eth0.
lvs_sync_daemon <INTERFACE> <VRRP_INSTANCE> [id <SYNC_ID>] [maxlen <LEN>] [port <PORT>] [ttl <TTL>] [group <IP ADDR>]
set up LVS Relevant contents of synchronization service. Can synchronize LVS Status information for.
INTERFACE: Specifies the interface of the synchronization service binding.
VRRP_INSTANCE: Specifies the of the synchronization service binding VRRP example.
id <SYNC_ID>: Specifies the used by the synchronization service SYNCID，Only the same SYNCID Will sync. The range is 0-255.
maxlen: Specifies the maximum length of the packet. The range is 1-65507
port: Specifies the used for synchronization UDP Port.
group: Specified multicast IP Address.
lvs_flush: stay keepalived When starting, refresh all existing LVS to configure.
vrrp_garp_master_delay 10: When converted to MASTER How many seconds is the delay in sending the second set of free messages ARP. The default is 5 s，0 Indicates that the second group of free will not be sent ARP.
vrrp_garp_master_repeat 1: When converted to MASTER Status, free information sent at one time in a group ARP quantity The default is 5.
vrrp_garp_lower_prio_delay 10: When MASTER How many seconds is the delay in sending the second group of free messages when receiving a lower priority notification ARP.
vrrp_garp_lower_prio_repeat 1: When MASTER When receiving a lower priority announcement, it is free to send it once in a group ARP quantity
vrrp_garp_master_refresh 60: When keepalived become MASTER Later, refresh for free ARP Minimum interval(Free will be sent again ARP). The default is 0, which means it will not be refreshed.
vrrp_garp_master_refresh_repeat 2: When keepalived become MASTER In the future, how many free messages will be sent for each refresh ARP. The default is 1.
vrrp_garp_interval 0.001: Two free messages sent on one interface ARP Delay between. It can be accurate to milliseconds. The default is 0.
vrrp_lower_prio_no_advert true|false: Default is false. If a low priority notification is received, no notification will be sent.
vrrp_version 2|3: Set default VRRP edition. The default is 2.
vrrp_check_unicast_src: In unicast mode, turn on VRRP Check the source address of the packet. The source address must be one of the unicast neighbors.
vrrp_skip_check_adv_addr: The default is not to skip the check. Check received VRRP All addresses in the advertisement may be time-consuming. Setting this command means that if the advertisement comes from the same address as the last advertisement received master Router, the check is not performed(Skip check).
vrrp_strict: Strictly abide by VRRP agreement. The following conditions will prevent startup Keepalived: 1. No, VIP Address. two. Unicast neighbors. three. stay VRRP In version 2 IPv6 Address.
vrrp_iptables: Do not add any iptables Rules. The default is add iptables Regular.
If vrrp Process or check If the process times out, you can use the following four options. Can make in BACKUP Stateful VRRP Instance becomes MASTER Status, even if MASTER The instance is still running. because MASTER or BACKUP The system is slow and cannot be handled in time VRRP Data packet.
vrrp_priority <-20 -- 19>: set up VRRP Priority of the process.
checker_priority <-20 -- 19>: set up checker Priority of the process.
vrrp_no_swap: vrrp Processes cannot be swapped.
checker_no_swap: checker Processes cannot be swapped.
script_user <username> [groupname]: Set the default user and group for running scripts. If not specified, the default user is keepalived_script(The user is required to exist)，Otherwise root User. default groupname with username.
enable_script_security: If any part of the script path is not root If users have writable permissions, they will not root Run the script as.
nopreempt The default is preemptive mode. If you use non preemptive mode, add it nopreempt
}
2.5.3 virtual router configuration
vrrp_instance <STRING> {
...
}
Parameter Description:
state MASTER|BACKUP: Specify this keepalived The initial state of the node.
interface eth0: vrrp Instance bound interface for sending VRRP Bag.
use_vmac [<VMAC_INTERFACE>]: Generate a sub interface at the specified interface, such as vrrp.51，Of this interface MAC The address is a multicast address, which is sent and received outward through the interface VRRP Bag.
vmac_xmit_base: Send and receive outward through the basic interface VRRP Packets, not through VMAC Interface.
native_ipv6: force VRRP Use instance IPV6.(When configured at the same time IPV4 and IPV6 When)
dont_track_primary: ignore VRRP The interface is not configured by default.
track_interface {
eth0
eth1 weight <-254-254>
...
}: If track If any of the interfaces fails, it will enter FAULT Status.
track_script {
<SCRIPT_NAME>
<SCRIPT_NAME> weight <-254-254>
}: Add a track script(vrrp_script Configured script.)
mcast_src_ip <IPADDR>: Specify the source to send multicast packets IP Address. The default is binding VRRP Master of the interface of the instance IP Address.
unicast_src_ip <IPADDR>: Specifies the source from which thin packets are sent IP Address. The default is binding VRRP Master of the interface of the instance IP Address.
version 2|3: Specifies the name of the file used by this instance VRRP edition.
unicast_peer {
<IPADDR>
...
}: Unicast transmission VRRP Notification, specifying unicast neighbors IP Address.
virtual_router_id 51: appoint VRRP example ID，The range is 0-255.
priority 100: Specify the priority, and the one with higher priority will become MASTER.
advert_int 1: Specify send VRRP Notice interval. The unit is seconds.
authentication {
auth_type PASS|AH: Specify the authentication method. PASS Simple password authentication(recommend),AH:IPSEC authentication(Not recommended).
auth_pass 1234: Specify the password to use for authentication. Up to 8 digits.
}
virtual_ipaddress {
<IPADDR>/<MASK> brd <IPADDR> dev <STRING> scope <SCOPE> label <LABEL>
192.168.200.17/24 dev eth1
192.168.200.18/24 dev eth2 label eth2:1
}: appoint VIP Address.
nopreempt: Set to no preemption. The default is preemption. When the high priority machine is restored, the low priority machine will be preempted MASTER，Without preemption, low priority machines are allowed to continue to become MASTER，Even if the high priority machines are online. If you want to use this function, the initialization status must be BACKUP.
preempt_delay: Set preemption delay. The unit is seconds and the range is 0---1000，The default is 0.Low priority found MASTER How many seconds after the start of preemption.
Notification script:
notify_master <STRING>|<QUOTED-STRING> [username [groupname]]
notify_backup <STRING>|<QUOTED-STRING> [username [groupname]]
notify_fault <STRING>|<QUOTED-STRING> [username [groupname]]
notify <STRING>|<QUOTED-STRING> [username [groupname]]
# Script executed when VRRP is stopped.
notify_stop <STRING>|<QUOTED-STRING> [username [groupname]]
smtp_alert
2.5.4 LVS configuration
virtual_server
virtual_server IP Port | virtual_server fwmark int | virtual_server group string {
delay_loop <INT>: Time interval of health examination.
lb_argo rr|wrr|lc|wlc|lblc|sh|dh: LVS Scheduling algorithm.
lb_kind NAT|DR|TUN: LVS pattern.
persistence_timeout 360: Persistence timeout, in seconds. The default is 6 minutes.
persistence_granularity: Granularity of persistent connections.
protocol TCP|UDP|SCTP: 4 Layer protocol.
ha_suspend: If virtual server of IP If the address is not set, the health check of the back-end server will not be performed.
virtualhost <STRING>: by HTTP_GET and SSL_GET Perform the virtual host to check. as virtualhost www.felix.com
sorry_server <IPADDR> <PORT>: Add a standby server. When all RS When all faults occur.
sorry_server_inhibit: take inhibit_on_failure Instructions apply to sorry_server Instructions.
alpha: stay keepalived At startup, it is assumed that all RS All down，And the health examination failed. Helps prevent false positives at startup. It is disabled by default.
omega: stay keepalived When terminated, the quorum_down The script defined by the directive.
quorum <INT>: Default 1. The total minimum weight of all surviving servers.
quorum_up <STRING>: When quorum Grow to meet quorum The script is executed when the defined value.
quorum_down <STRING>: When quorum Reduce to dissatisfaction quorum The script is executed when the defined value.
}
real_server
real_server IP Port {
weight <INT>: Assign weights to the server. The default is 1.
inhibit_on_failure: When the server health check fails, it is weight Set to 0 instead of from Virtual Server Remove from.
notify_up <STRING>: Script to execute when the server health check is successful.
notify_down <STRING>: Script to execute when the server health check fails.
uthreshold <INT>: The maximum number of connections to this server.
lthreshold <INT>: The minimum number of connections to this server.
}
2.5.5 example
global_defs {
router_id LVS_Server Specify the that identifies the machine route_id
}
vrrp_instance VI_1 {
state MASTER Specify this keepalived Initial state of node
interface ens8 vrrp Instance bound interface for sending VRRP package
virtual_router_id 51 appoint VRRP example ID
priority 150 Specify the priority, and the one with higher priority will become MASTER
nopreempt Set to no preemption. The default is preemptive
advert_int 1 advert_int 1
authentication {
auth_type PASS Specify authentication method
auth_pass password Specify the password to use for authentication.
}
virtual_ipaddress {
192.168.1.217 dev ens8 appoint VIP address
}
}
virtual_server 192.168.1.217 443 {
delay_loop 3 delay_loop
lvs_sched rr LVS Scheduling algorithm
lvs_method DR LVS pattern
protocol TCP 4 Layer protocol
real_server 192.168.1.211 443 {
weight 1
TCP_CHECK {
connect_port 443
connect_timeout 3
nb_get_retry 3 get Number of attempts
delay_before_retry 10 How long is the delay before trying
}
}
real_server 192.168.1.212 443 {
weight 1
TCP_CHECK {
connect_port 443
connect_timeout 3
nb_get_retry 3
delay_before_retry 10
}
}
}
virtual_server 192.168.1.217 80 {
delay_loop 3
lvs_sched rr
lvs_method DR
protocol TCP
real_server 192.168.1.211 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 10
}
}
}
real_server 192.168.1.212 80 {
weight 1 Adjust priority. The default is 2
TCP_CHECK {
connect_port 80 Connected port
connect_timeout 3 Connection timeout. The default is 5 s.
nb_get_retry 3 get Number of attempts.
delay_before_retry 10
}
}
}
3. HAProxy+keepalived to achieve high cluster availability
3.1 environmental planning
| ip | explain |
|---|---|
| 192.168.213.11 | Haproxy + keepalived (Master) |
| 192.168.213.12 | Haproxy + keepalived (Slave) |
| 192.168.213.13 | Web01 |
| 192.168.213.14 | Web02 |
| 192.168.213.100 | VIP |
3.2 HAProxy configuration
# The two nodes are consistent
global
chroot /apps/haproxy
daemon
maxconn 100000
stats socket /var/lib/haproxy/haproxy1.sock mode 600 level admin process 1
stats socket /var/lib/haproxy/haproxy2.sock mode 600 level admin process 2
user haproxy
group haproxy
nbproc 3
pidfile /var/lib/haproxy/haproxy.pid
log 127.0.0.1 local2 info
defaults
mode http
timeout connect 5000ms
timeout client 50000ms
timeout server 50000ms
stats uri /status
stats auth zp:123456
listen stats
mode http
bind 0.0.0.0:9999
stats enable
log global
stats uri /haproxy-status
stats auth haadmin:123456
frontend http-in
bind *:80
default_backend servers
backend servers
server web01 192.168.213.13:80 cookie web01 check inter 1500 rise 3 fall 3 weight 1
server web02 192.168.213.14:80 cookie web02 check inter 1500 rise 3 fall 3 weight 1
3.3 keepalived configuration
# master
! Configuration File for keepalived master
global_defs {
smtp_connect_timeout 30
router_id LVS_DEVEL01
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.213.100
}
}
# SLAVE
! Configuration File for keepalived slave
global_defs {
smtp_connect_timeout 30
router_id LVS_DEVEL02
}
vrrp_instance VI_1 {
state SLAVE
interface ens33
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.213.100
}
}
3.4 verification
# Access test
[root@web01 ~]# curl 192.168.213.100
web01
[root@web01 ~]# curl 192.168.213.100
web01
[root@web01 ~]# curl 192.168.213.100
web02
[root@web01 ~]# curl 192.168.213.100
web02
# High availability test
[root@haproxykeepalived_1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:f0:a5:10 brd ff:ff:ff:ff:ff:ff
inet 192.168.213.11/24 brd 192.168.213.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.213.100/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::8625:ae97:6a35:c349/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@haproxykeepalived_2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:86:ba:a9 brd ff:ff:ff:ff:ff:ff
inet 192.168.213.12/24 brd 192.168.213.255 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe86:baa9/64 scope link
valid_lft forever preferred_lft forever
[root@haproxykeepalived_1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:f0:a5:10 brd ff:ff:ff:ff:ff:ff
inet 192.168.213.11/24 brd 192.168.213.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.213.100/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::8625:ae97:6a35:c349/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@haproxykeepalived_1 ~]# systemctl stop keepalived.service
[root@haproxykeepalived_1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:f0:a5:10 brd ff:ff:ff:ff:ff:ff
inet 192.168.213.11/24 brd 192.168.213.255 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::8625:ae97:6a35:c349/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@haproxykeepalived_2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:86:ba:a9 brd ff:ff:ff:ff:ff:ff
inet 192.168.213.12/24 brd 192.168.213.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.213.100/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe86:baa9/64 scope link
valid_lft forever preferred_lft forever
[root@web01 ~]# curl 192.168.213.100
web01
[root@web01 ~]# curl 192.168.213.100
web02
[root@web01 ~]# curl 192.168.213.100
web02
[root@web01 ~]# curl 192.168.213.100
web01