Tags (space separated): building big data platform

1.1: introduction to system environment

System: RHEL7.7X64

cat /etc/hosts 
---
192.168.100.21   rhel01.flyfish
192.168.100.22   rhel02.flyfish
192.168.100.23   rhel03.flyfish
192.168.100.24   rhel04.flyfish

-----

Only three machines are configured in this installation: 192.168.100.21 to 192.168.100.23 

1.2: turn off firewalld, iptables and selinux

 yum install -y firewall* iptable* 
  (1) Close firewalld
 systemctl stop firewalld.service
 systemctl disable firewalld.service
 firewall-cmd --state
 (2) Close iptables 
 iptables -F
 systemctl stop iptables.service 
 service iptables save
 systemctl disable iptables.service 
 (3) Prohibit selinux 
  sed -i '/SELINUX/s/enforcing/disabled/' /etc/selinux/config  
  setenforce 0
  getenforce 0
  sestatus
  //Restart the machine
   reboot

1.3: Keyless login configuration

Do root authentication without key
 SSH keygen -- always hit enter until the end
 cat id_rsa.pub >> authorized_keys 
 Import all public keys into authorized_keys and distribute them to all machines under. ssh /
 Then test

1.4 configure jdk1.8 environment (all machines are configured)

1): Uninstall the original jdk
rpm -e java-1.8.0-openjdk-headless-1.8.0.242.b08-0.el7_7.x86_64 javapackages-tools-3.4.1-11.el7.noarch tzdata-java-2019c-1.el7.noarch python-javapackages-3.4.1-11.el7.noarch postgresql-jdbc-javadoc-9.2.1002-6.el7_5.noarch --nodeps

2) install jdk:

rpm -ivh jdk-8u162-linux-x64.rpm

vim /etc/profile 

//Finally, add:

### JDK ####

export JAVA_HOME=/usr/java/jdk1.8.0_162
export CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$HOME/bin:$JAVA_HOME/bin
---

java -version 

1.5: chrony time synchronization server

All machines yum install chrony*
1. rhel01.flyfish:
vim /etc/chrony.conf increase
----
//Comment out the related default server
server ntp1.aliyun.com iburst
allow 192.168.100.0.0/16
service chronyd start
chkconfig chronyd on 
2. rhel02.flyfish---rhel03.flyfish synchronization rhel01.flyfish
vim /etc/chrony.conf
----
//Comment out the default server
server 192.168.100.21 iburst
----
service chronyd start 
chkconfig chronyd on 

1.6: install the malidb server database

1)install mariadb-server
yum install -y mariadb*

2) start-up mariadb-server

service mariadb start 
chkcofig mariadb on 

3) Configuration database password:

mysql_secure_installation

4) Sign in mariadb-server data base

   mysql -uroot -pflyfish225

1.8 system parameter configuration

vim /etc/sysctl.conf

---
//Execute the following command for the parameters to take effect:
##Kernel default parameters 
kernel.sysrq = 0 
kernel.core_uses_pid = 1 
kernel.msgmnb = 65536 
kernel.msgmax = 65536 
kernel.shmmax = 68719476736 
kernel.shmall = 4294967296 
##Number of open files parameter (20 * 1024 * 1024) 
fs.file-max= 20971520 
##WEB Server parameters 
net.ipv4.tcp_tw_reuse=1 
net.ipv4.tcp_tw_recycle=1 
net.ipv4.tcp_fin_timeout=30 
net.ipv4.tcp_keepalive_time=1200 
net.ipv4.ip_local_port_range = 1024 65535 
net.ipv4.tcp_rmem=4096 87380 8388608 
net.ipv4.tcp_wmem=4096 87380 8388608 
net.ipv4.tcp_max_syn_backlog=8192 
net.ipv4.tcp_max_tw_buckets = 5000 
##TCP supplementary parameters 
net.ipv4.ip_forward = 0 
net.ipv4.conf.default.rp_filter = 1 
net.ipv4.conf.default.accept_source_route = 0 
net.ipv4.tcp_syncookies = 1 
net.ipv4.tcp_sack = 1 
net.ipv4.tcp_window_scaling = 1 
net.core.wmem_default = 8388608 
net.core.rmem_default = 8388608 
net.core.rmem_max = 16777216 
net.core.wmem_max = 16777216 
net.core.netdev_max_backlog = 262144 
net.ipv4.tcp_max_orphans = 3276800 
net.ipv4.tcp_timestamps = 0 
net.ipv4.tcp_synack_retries = 1 
net.ipv4.tcp_syn_retries = 1 
net.ipv4.tcp_mem = 94500000 915000000 927000000 
##Disable ipv6 
net.ipv6.conf.all.disable_ipv6 =1 
net.ipv6.conf.default.disable_ipv6 =1 
##swap usage optimization 
vm.swappiness=0 
# sysctl -p 
//Synchronize to all machines

1.7 modify the number of file handles

vim /etc/security/limits.conf
---

*               soft    nofile          65535
*               hard    nofile          1029345
*               soft    nproc           unlimited
*               hard    nproc           unlimited
*               soft    memlock         unlimited
*               hard    memlock         unlimited

---

2: Installation of CDH5.16.2 platform

2.1: install configuration dependency package (all nodes are installed)

yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb postgresql* portmap mod_ssl openssl-devel python-psycopg2 MySQL-python

2.2: install cloudera server and change configuration (install all nodes)

rpm -ivh cloudera-manager*
rpm -ivh enterprise-debuginfo-5.14.3-1.cm5143.p0.4.el7.x86_64.rpm

2.3: create CM Library in MariaDB server

mysql -uroot -pflyfish225.com
mysql> create database cmf character set utf8;
mysql> grant all privileges on cmf.* to 'cmf'@'%' identified by 'cmf_1234' with grant option;
mysql> grant all privileges on cmf.* to 'cmf'@'node-01.flyfish' identified by 'cmf_1234' with grant option;
mysql> flush privileges;

2.4: install offline cdh parcel-repo

#cd cdh5.16.2
#cp -p CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel* /opt/cloudera/parcel-repo/
#cp -p manifest.json /opt/cloudera/parcel-repo/
#cd /opt/cloudera/parcel-repo
#mv CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha1 CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha   
chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo/ (All hosts need to execute this command)

2.5 configure jdbc Driver

tar -zxvf mysql-connector-java-5.1.44.tar.gz
cd mysql-connector-java-5.1.44
mv mysql-connector-java-5.1.44-bin.jar /usr/share/java
cd /usr/share/java
mv mysql-connector-java-5.1.44-bin.jar mysql-connector-java.jar
chmod 777 mysql-connector-java.jar

2.6 registration generation table of configuration data

1)backups db Files that generate tables
cp /etc/cloudera-scm-server/db.properties /etc/cloudera-scm-server/db.propertiesbak 

2)Generate table

/usr/share/cmf/schema/scm_prepare_database.sh mysql cmf cmf cmf_1234

2.7: modify config.ini file

vim /etc/cloudera-scm-agent/config.ini
---
server_host=192.168.100.21
---

scp /etc/cloudera-scm-agent/config.ini root@rhel02.flyfish:/etc/cloudera-scm-agent/

scp /etc/cloudera-scm-agent/config.ini root@rhel03:/etc/cloudera-scm-agent/

2.8: start CM-SERVER

cd /etc/init.d/
./cloudera-scm-server start

View log:
cd /var/log/cloudera-scm-server

tail -100f cloudera-scm-server.log

Check that there are 7180 ports to start, indicating that CM management server is started

netstat -nultp |grep 7180

Open with browser web:

http://192.168.100.21:7180

start-up agent End:

cd /etc/init.d/
./cloudera-scm-agent start

3: Assign hosts and assign roles

All nodes execute:
#echo never > /sys/kernel/mm/transparent_hugepage/defrag
#echo never > /sys/kernel/mm/transparent_hugepage/enabled
vim /etc/rc.local 
//Add at the bottom:
---
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
---
//Synchronize all machines

scp /etc/rc.d/rc.local root@rhel02.flyfish:/etc
scp /etc/rc.d/rc.local root@rhel03.flyfish:/etc

mysql Authorized configuration

//Database authorization:
mysql -uroot -pflyfish225
---
//Create the libraries needed for hive and authorize:
mysql >create database cdh_hive default character set latin1; 
//The character must be Latin (latin1), if uft8 may report an error Specified key was too long; max key length is 767 bytes 
mysql >grant all privileges on cdh_hive.* to 'hive'@'%' identified by 'Hive_123' with grant option;
mysql >grant all privileges on cdh_hive.* to 'hive'@'rhel01.flyfish' identified by 'Hive_123' with grant option;
mysql >flush privileges; 

//Create the library needed by oozie
mysql >create database cdh_oozie default character set utf8; 
mysql >grant all privileges on cdh_oozie.* to 'oozie'@'%' identified by 'Oozie_123' with grant option;
mysql >grant all privileges on cdh_oozie.* to 'oozie'@'rhel01.flyfish' identified by 'Oozie_123' with grant option;
mysql >flush privileges; 

//To create a library for hue:
mysql >create database cdh_hue default character set utf8; 
mysql >grant all privileges on cdh_hue.* to 'hue'@'%' identified by 'Hue_123' with grant option;
mysql >grant all privileges on cdh_hue.* to 'hue'@'rhel01.flyfish' identified by 'Hue_123' with grant option;
mysql >flush privileges; 

//Create a library of Activity Monitor at the monitoring end of CM
Activity Monitor:
mysql >create database cdh_am default character set utf8; 
mysql >grant all privileges on cdh_am.* to 'am'@'%' identified by 'Am_123' with grant option;
mysql >grant all privileges on cdh_am.* to 'am'@'rhel01.flyfish' identified by 'Am_123' with grant option;
mysql >flush privileges;

//Create RM's monitoring end Reports Manager Library
Reports Manager
mysql >create database cdh_rm default character set utf8; 
mysql >grant all privileges on cdh_rm.* to 'rm'@'%' identified by 'Rm_123' with grant option;
mysql >grant all privileges on cdh_rm.* to 'rm'@'rhel01.flyfish' identified by 'Rm_123' with grant option;
mysql >flush privileges;