1. rsync overview
1.rsync is an open source, fast, multi-functional and excellent tool that can realize full and incremental synchronous backup of local or remote data. It can realize the backup and migration characteristics of data without changing the attribute information of the original data. In short, it is replication, which can be used for local replication or replication between users across hosts (ssh/rsync host synchronization)
2.rsync software is suitable for unix/linux/windows and other operating system platforms. It provides a large number of parameters to control all aspects of its behavior, which is very flexible
3.rsync software is famous for its delta transfer algorithm
4.rsync operation mode: c/s, rsync listening port: 873, the latest version is 3.1 3. Maintenance by Wayne Davison
- Configure rsync source server
2.1 basic ideas
Basic idea of configuring backup source
rsync synchronization source: refers to the remote server of the backup operation, also known as the backup source
Create rsyncd Conf configuration file and independent account file; The service account remote login uses the related functions of rsync, and the service account is user-defined
Enable the – daemon mode of rsync
2.2 application examples
1. Configuration file - rsyncd conf
It needs to be established manually, and the syntax is similar to Samba configuration
auth users and secrets file are configured for authentication. If not added, it is anonymous
2. Role of Rsync account file
The record format of "user name: password" is adopted, with one user record per line; Independent account data, independent of the system account
Different synchronization strategies / methods can be used when corresponding profile modules (different account + function modules) are required
When using synchronization, we need to specify the account + function module to specify our synchronization strategy
3. Enable rsync service
Service provided by – daemon alone
Execute kill $(cat /var/run/rsyncd.pid) to shut down the rsync service
2.3 two representations of configuration source
Format I
User name @ host address:: shared module name
rsync -avz backuper@192.168.35.40::wwwroot /root
Format II
rsyne: / / username @ host address / shared module name
rsync -avz rsync:/ /backuper@192.168.152.130::/wwwroot /root
- Backup tool rsync
3.1 synchronization mode
1. Full backup
All original data is transmitted
Send the original document and the new document together
Full replication, low efficiency
2. Incremental backup
Before transmitting data, compare the data you have with the data I have through some algorithms to transmit different data through the network
Incremental replication, high efficiency
3.2 rsync command
Basic format: rsync [option] Original position target position
4. Deploy rsync synchronization
4.1 rsync local replication
4.1. 1. Construction environment
rsync 192.168.152.130
client 192.168.152.129
To test local replication, first use a virtual machine with address 130 as the test
4.1. 2. Construction process
download rsync And start: [root@rsync ~]#rpm -q rsync [root@rsync ~]#yum -y install rsync ##Install rsync [root@rsync ~]# mkdir /aaa [root@rsync ~]# mkdir /bbb [root@rsync ~]# mkdir /ccc [root@rsync ~]# cd / [root@rsync /]# ls aaa bin dev lib mnt root srv usr abc boot etc lib64 opt run sys var bbb ccc home media proc sbin tmp [root@rsync ~]# cd /aaa [root@rsync aaa]# touch 1.txt [root@rsync aaa]# ls 1.txt [root@rsync aaa]# rsync -avz /aaa/ /bbb #Copy the files in directory aaa to directory bbb sending incremental file list ./ 1.txt sent 103 bytes received 38 bytes 282.00 bytes/sec total size is 0 speedup is 0.00 [root@rsync aaa]# rsync -avz /aaa /ccc #Copy the files in directory aaac and directory aaa to directory ccc sending incremental file list aaa/ aaa/1.txt sent 115 bytes received 39 bytes 308.00 bytes/sec total size is 0 speedup is 0.00 #The difference between rsync and cp is that the cp command copies the directory itself whether it is followed by / or not [root@rsync aaa]# cp -a /aaa /opt [root@rsync aaa]# ls /opt aaa httpd-2.4.6-95.el7.centos.x86_64.rpm rh [root@rsync aaa]# rm -rf /opt/aaa/ [root@rsync aaa]# cp -a /aaa/ /opt [root@rsync aaa]# ls /opt/ aaa httpd-2.4.6-95.el7.centos.x86_64.rpm rh [root@rsync aaa]#
4.2 rsync remote replication
4.2. 1. Construction environment
rsync 192.168.152.130 client 192.168.152.129 4.2.2 Construction process download rsync And start: [root@rsync ~]#rpm -q rsync [root@rsync ~]#yum -y install rsync ##Install rsync to configure rsync The server: [root@rsync ~]# vim /etc/rsyncd.conf uid = nobody #root gid = nobody #root use chroot = yes #Locked in the source directory address = 192.168.152.130 #Listening address port 873 #Listen on port tcp/udp 873. You can view it through cat /etc/services | grep rsync log file = /var/log/rsyncd.log #Log file location pid file = /var/run/rsyncd.pid #The file location where the process ID is stored hosts allow = 192.168.152.0/24 #Allowed client addresses [wwwroot] ##First shared module path = /var/www/html #The actual path to the source directory comment = Document Root of www.ljm.com read only = yes #Is it read-only dont comperss = *.gz *.bz2 *.tgz *.zip *.rar *.z #File types that are no longer compressed during synchronization auth users = backuper #Authorized accounts. Multiple accounts are separated by spaces secrets file = /etc/user.db #Data file for storing account information To create a data file for a backup account: [root@rsync ~]# vim /etc/user.db [root@rsync ~]# cat /etc/user.db backuper:abc [root@rsync ~]# chmod 600 /etc/user.db #Ensure that all users have read permission to the source directory / var/www/html #Permission must be given to 600, otherwise an error will be reported Create shared directory: [root@rsync ~]# mkdir -p /var/www/html [root@rsync ~]# chmod +r /var/www/html/ [root@rsync ~]# ls -ld /var/www/html/ drwxr-xr-x. 2 root root 24 11 June 17, 2020 /var/www/html/ [root@rsync ~]# rsync --daemon #Start rsync service program [root@rsync ~]# netstat -antp | grep rsync tcp 0 0 192.168.152.130:873 0.0.0.0:* LISTEN 18562/rsync [root@rsync ~]#
1. Interactive: verify whether the client side realizes remote sharing
Create directory abc under the root directory of client:
[root@client /]# rsync -avz backuper@192.168.152.130::wwwroot /abc/
Password:
#Note that the password here is not the password of the virtual machine, but the user just created The password for DB is abc123
receiving incremental file list
./
1.txt
index.html
sent 65 bytes received 204 bytes 3.93 bytes/sec
total size is 38 speedup is 0.14
[root@client /]# cd /abc/
[root@client abc]# ls
1.txt index.html
[ root@client abc]# cat 1. Txt # you can see that it has been synchronized
hello world
hello world
Non interactive: [root@client /]# vim /etc/server.pass [root@client /]# cat /etc/server.pass abc123 [root@client /]# chmod 600 /etc/server.pass [root@client /]# cd /abc [root@client abc]# rsync -avz --password-file=/etc/server.pass backuper@192.168.152.130::wwwroot /abc receiving incremental file list ./ 1.txt sent 46 bytes received 150 bytes 18.67 bytes/sec total size is 38 speedup is 0.19 [root@client abc]# ls 1.txt index.html Here is a bold attempt to delete rsync See the contents of the shared directory on the client side client What content can the client synchronize to: [root@rsync ~]# rm -rf /var/www/html/* [root@rsync ~]# ls /var/www/html/ [root@rsync ~]# client Synchronize again: the current status is rsync There's nothing on the end, but client End presence 1.txt file [root@client abc]# rsync -avz --password-file=/etc/server.pass backuper@192.168.152.130::wwwroot /abc receiving incremental file list ./ sent 27 bytes received 47 bytes 6.43 bytes/sec total size is 0 speedup is 0.00 #You can see that there is no synchronization. There is a logical difference here. Since you synchronize others, you should also delete 1 Txt file, but he didn't delete it #What we want to achieve is that the synchronization should be local and also delete 1 Txt, plus -- delete #--delete deletes those with the target location but not the source address, in short, those with the client but not rsync [root@client abc]# rsync -avz --delete --password-file=/etc/server.pass backuper@192.168.152.130::wwwroot /abc receiving incremental file list deleting index.html deleting 1.txt sent 20 bytes received 40 bytes 5.22 bytes/sec total size is 0 speedup is 0.00 [root@client abc]# ls
4.3 optimization of Rsync synchronization
1. Lack of regular synchronization
The backup time is fixed, with obvious delay and poor real-time performance
When the synchronization source does not change for a long time, intensive periodic tasks are unnecessary
It needs to be triggered manually, shell script or crontab
2. Advantages of real-time synchronization
Once the synchronization source changes, start the backup immediately
No backup is performed as long as the synchronization source remains unchanged
- rsync combined with inotify real-time synchronization
5.1 inotify overview
You can monitor file system changes and respond to notifications
Inotify tools need to be installed to use inotify mechanism to provide inotifywait and inotifywatch auxiliary tools to monitor and summarize changes
1.adjustment inotify Kernel parameters (optimization) /etc/sysctl.conf(Kernel parameter profile) inotifywait: #It is used for continuous monitoring and real-time output of results inotifywatch: #It is used for short-term monitoring and output results after the task is completed max_queue_events #Monitor event queue size max_user_instances #Maximum number of monitoring instances max_user_watches #Maximum number of monitoring files per instance 2,Commands for continuous monitoring and real-time output of monitoring results: inotifywait Format: inotifywait [parameter] Common parameters explain -m Continuous monitoring -r Recursively monitor all child objects -q Simplified output information -e Specify which event types to monitor
experiment
rsync+inotify real-time synchronization
master
// [root@master ~]# vim /etc/rsyncd.conf uid = root gid = root use chroot = yes address = 192.168.142.143 port 873 log file = /var/log/rsyncd.log pid file = /var/run/rsyncd.pid hosts allow = 192.168.142.0/24 [wwwroot] path = /var/www/html comment = Document Root of www.lic.com read only = no dont comperss = *.gz *.bz2 *.tgz *.zip *.rar *.z auth users = zhangsan secrets file = /etc/rsyncd_users.db [root@master html]# kill `cat /var/run/rsyncd.pid` [root@master html]# netstat -anpt | grep rsync [root@master html]# rsync --daemon [root@master html]# netstat -anpt | grep rsync tcp 0 0 192.168.142.143:873 0.0.0.0:* LISTEN 28554/rsync
slaver
// [root@slaver ~]# yum -y install gcc gcc-c++ [root@slaver ~]# cd /opt [root@slaver opt]# rz -E rz waiting to receive. [root@slaver opt]# tar xf inotify-tools-3.14.tar.gz [root@slaver opt]# ls inotify-tools-3.14 inotify-tools-3.14.tar.gz rh test [root@slaver opt]# cd inotify-tools-3.14 [root@slaver inotify-tools-3.14]# ./configure && make && make install
Adjust kernel parameters
// [root@slaver inotify-tools-3.14]# vim /etc/sysctl.conf fs.inotify.max_queued_events = 32768 #Monitoring time queue, the default is 16384 fs.inotify.max_user_instances = 1024 #The maximum number of monitoring instances is 128 by default fs.inotify.max_user_watches = 1048576 #The maximum number of monitoring files per instance is 8192 by default #When the amount of directory and file data to be monitored is large or changes frequently, it is recommended to increase the parameter value
Edit scripts that automatically trigger synchronization
//
[root@slaver ~]# vim /opt/inotify.sh
#!/bin/bash
INOTIFY_CMD="inotifywait -mrq -e create,delete,move,modify,attrib /opt/test/"
RSYNC_CMD="rsync -azH --delete --password-file=/etc/server.pass /opt/test/ zhangsan@192.168.142.143::wwwroot"
$INOTIFY_CMD | while read DIRECTORY EVENT FILE
do
if [ $(pgrep rsync | wc -l) -le 0 ] ; then
$RSYNC_CMD
fi
done
// [root@slaver ~]#cd /opt/ [root@slaver opt]# chmod +x inotify.sh [root@slaver opt]# ./inotify.sh [root@slaver opt]# cd /opt/test [root@slaver test]# touch ccc.html [root@slaver test]# rm -rf aaa.html
