使用阿里云服务器进行CDH部署学习

1 [root@hadoop001 ~]# wget http://gosspublic.alicdn.com/ossutil/1.6.3/ossutil64
2 --2019-06-29 09:48:21-- http://gosspublic.alicdn.com/ossutil/1.6.3/ossutil64
3 Resolving gosspublic.alicdn.com (gosspublic.alicdn.com)... 205.204.104.233, 205.204.104.242
4 Connecting to gosspublic.alicdn.com (gosspublic.alicdn.com)|205.204.104.233|:80... connected.
5 HTTP request sent, awaiting response... 200 OK
6 Length: 9741374 (9.3M) [application/octet-stream]
7 Saving to: ‘ossutil64’
8
9 100%[=========================================================================================================================================================>] 9,741,374 3.21MB/s in 2.9s
10
11 2019-06-29 09:48:25 (3.21 MB/s) - ‘ossutil64’ saved [9741374/9741374]

1 [root@hadoop001 cdh5.16.1]# ll
2 total 3614496
3 -rw-r--r-- 1 root root 2127506677 Jun 29 10:04 CDH-5.16.1-1.cdh5.16.1.p0.3-el7.parcel
4 -rw-r--r-- 1 root root 41 Jun 29 10:04 CDH-5.16.1-1.cdh5.16.1.p0.3-el7.parcel.sha1
5 -rw-r--r-- 1 root root 841524318 Jun 29 10:04 cloudera-manager-centos7-cm5.16.1_x86_64.tar.gz
6 -rw-r--r-- 1 root root 173271626 Jun 29 10:04 jdk-8u45-linux-x64.gz
7 -rw-r--r-- 1 root root 66538 Jun 29 10:04 manifest.json
8 -rw-r--r-- 1 root root 548193637 Jun 29 10:04 mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz
9 -rw-r--r-- 1 root root 1007502 Jun 29 10:05 mysql-connector-java-5.1.47.jar
10 -rw-r--r-- 1 root root 9641827 Jun 29 10:05 ossutil64

1 [root@hadoop001 .ssh]# ssh-keygen -t rsa
2 Generating public/private rsa key pair.
3 Enter file in which to save the key (/root/.ssh/id_rsa):
4 Enter passphrase (empty for no passphrase):
5 Enter same passphrase again:
6 Your identification has been saved in /root/.ssh/id_rsa.
7 Your public key has been saved in /root/.ssh/id_rsa.pub.
8 The key fingerprint is:
9 SHA256:QLf04jdE2pLmSEyWlbg/7YKlmuVFAeNAEZE24HKePpA root@hadoop001
10 The key's randomart image is:
11 +---[RSA 2048]----+
12 | .o*=*++.. |
13 | . +Bo=.B |
14 |. o. .=.O + |
15 | = . ..* = |
16 |E o ..S.o |
17 | o .+... |
18 | o .+.o |
19 | . +o.. . |
20 | o.. . |
21 +----[SHA256]-----+

1 [root@hadoop001 .ssh]# cat id_rsa.pub >> /root/authorized_keys
2 [root@hadoop001 .ssh]# cd ..
3 [root@hadoop001 ~]# ll
4 total 9524
5 -rw-r--r-- 1 root root 396 Jun 29 11:05 authorized_keys
6 drwxr-xr-x 2 root root 4096 Jun 29 10:05 cdh5.16.1
7 -rwxr-xr-x 1 root root 9741374 Jun 20 15:57 ossutil64
8 [root@hadoop001 ~]# scp id_rsa.pub hadoop003:/root/.ssh/
9 root@hadoop003's password:
10 id_rsa.pub: No such file or directory
11 [root@hadoop001 ~]# scp authorized_keys hadoop003:/root/.ssh/
12 root@hadoop003's password:
13 authorized_keys 100% 396 1.5MB/s 00:00
14 [root@hadoop001 ~]# scp authorized_keys hadoop004:/root/.ssh/
15 root@hadoop004's password:
16 authorized_keys

1 [root@hadoop001 ~]# vi /etc/hosts

1 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
2 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
3 # cdh
4 172.31.188.179 hadoop001
5 172.31.188.180 hadoop002
6 172.31.188.181 hadoop003
7 172.31.188.182 hadoop004
8 ~
9 ~

1 [root@hadoop001 cdh5.16.1]# ping hadoop002
2 PING hadoop002 (172.31.188.180) 56(84) bytes of data.
3 64 bytes from hadoop002 (172.31.188.180): icmp_seq=1 ttl=64 time=0.146 ms
4 ^Z
5 [2]+ Stopped ping hadoop002
6 [root@hadoop001 cdh5.16.1]# ping hadoop003
7 PING hadoop003 (172.31.188.181) 56(84) bytes of data.
8 64 bytes from hadoop003 (172.31.188.181): icmp_seq=1 ttl=64 time=0.295 ms
9 ^Z
10 [3]+ Stopped ping hadoop003
11 [root@hadoop001 cdh5.16.1]# ping hadoop004
12 PING hadoop004 (172.31.188.182) 56(84) bytes of data.
13 64 bytes from hadoop004 (172.31.188.182): icmp_seq=1 ttl=64 time=0.331 ms
14 ^Z
15 [4]+ Stopped ping hadoop004

1 [root@hadoop001 ~]# scp /etc/hosts hadoop002:/etc
2 hosts 100% 264 1.2MB/s 00:00
3 [root@hadoop001 ~]# scp /etc/hosts hadoop003:/etc
4 hosts 100% 264 1.2MB/s 00:00
5 [root@hadoop001 ~]# scp /etc/hosts hadoop004:/etc
6 hosts 100% 264 1.0MB/s 00:00

1 [root@hadoop001 ~]# tar -xzvf cdh5.16.1/jdk-8u45-linux-x64.gz -C /usr/java/

1 [root@hadoop001 java]# chown -R root:root jdk1.8.0_45/

1 [root@hadoop001 ~]# source /etc/profile
2 [root@hadoop001 ~]# java -version
3 java version "1.8.0_45"
4 Java(TM) SE Runtime Environment (build 1.8.0_45-b14)
5 Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode)

1 [root@hadoop001 ~]# scp -r /usr/java/ hadoop002:/usr/java/

[root@hadoop001 ~]# scp /etc/profile hadoop002:/etc
profile 100% 1873 7.0MB/s 00:00
[root@hadoop001 ~]# scp /etc/profile hadoop003:/etc
profile 100% 1873 7.2MB/s 00:00
[root@hadoop001 ~]# scp /etc/profile hadoop004:/etc
profile

1 tar -xzvf /root/cdh5.16.1/mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz -C /usr

1 [root@hadoop001 local]# tar -xzvf /root/cdh5.16.1/mysql-5.7.11-linux-glibc2.5-x86_64.tar.gz -C /usr/local/
2 [root@hadoop001 local]# mv mysql-5.7.11-linux-glibc2.5-x86_64 mysql
3 [root@hadoop001 local]# ll
4 total 48
5 drwxr-xr-x 6 root root 4096 Jun 29 14:43 aegis
6 drwxr-xr-x. 2 root root 4096 Nov 5 2016 bin
7 drwxr-xr-x. 2 root root 4096 Nov 5 2016 etc
8 drwxr-xr-x. 2 root root 4096 Nov 5 2016 games
9 drwxr-xr-x. 2 root root 4096 Nov 5 2016 include
10 drwxr-xr-x. 2 root root 4096 Nov 5 2016 lib
11 drwxr-xr-x. 2 root root 4096 Nov 5 2016 lib64
12 drwxr-xr-x. 2 root root 4096 Nov 5 2016 libexec
13 drwxr-xr-x 9 7161 wheel 4096 Feb 2 2016 mysql
14 drwxr-xr-x. 2 root root 4096 Nov 5 2016 sbin
15 drwxr-xr-x. 6 root root 4096 Jun 29 09:42 share
16 drwxr-xr-x. 2 root root 4096 Nov 5 2016 src
17 [root@hadoop001 local]# mkdir mysql/arch mysql/data mysql/tmp
18 [root@hadoop001 local]# vi /etc/my.cnf
19 [root@hadoop001 local]# cat /etc/my.cnf
20 [client]
21 port = 3306
22 socket = /usr/local/mysql/data/mysql.sock
23 default-character-set=utf8mb4
24
25 [mysqld]
26 port = 3306
27 socket = /usr/local/mysql/data/mysql.sock
28
29 skip-slave-start
30
31 skip-external-locking
32 key_buffer_size = 256M
33 sort_buffer_size = 2M
34 read_buffer_size = 2M
35 read_rnd_buffer_size = 4M
36 query_cache_size= 32M
37 max_allowed_packet = 16M
38 myisam_sort_buffer_size=128M
39 tmp_table_size=32M
40
41 table_open_cache = 512
42 thread_cache_size = 8
43 wait_timeout = 86400
44 interactive_timeout = 86400
45 max_cOnnections= 600
46
47 # Try number of CPU's*2 for thread_concurrency
48 #thread_cOncurrency= 32
49
50 #isolation level and default engine
51 default-storage-engine = INNODB
52 transaction-isolation = READ-COMMITTED
53
54 server-id = 1739
55 basedir = /usr/local/mysql
56 datadir = /usr/local/mysql/data
57 pid-file = /usr/local/mysql/data/hostname.pid
58
59 #open performance schema
60 log-warnings
61 sysdate-is-now
62
63 binlog_format = ROW
64 log_bin_trust_function_creators=1
65 log-error = /usr/local/mysql/data/hostname.err
66 log-bin = /usr/local/mysql/arch/mysql-bin
67 expire_logs_days = 7
68
69 innodb_write_io_threads=16
70
71 relay-log = /usr/local/mysql/relay_log/relay-log
72 relay-log-index = /usr/local/mysql/relay_log/relay-log.index
73 relay_log_info_file= /usr/local/mysql/relay_log/relay-log.info
74
75 log_slave_updates=1
76 gtid_mode=OFF
77 enforce_gtid_cOnsistency=OFF
78
79 # slave
80 slave-parallel-type=LOGICAL_CLOCK
81 slave-parallel-workers=4
82 master_info_repository=TABLE
83 relay_log_info_repository=TABLE
84 relay_log_recovery=ON
85
86 #other logs
87 #general_log =1
88 #general_log_file = /usr/local/mysql/data/general_log.err
89 #slow_query_log=1
90 #slow_query_log_file=/usr/local/mysql/data/slow_log.err
91
92 #for replication slave
93 sync_binlog = 500
94
95
96 #for innodb options
97 innodb_data_home_dir = /usr/local/mysql/data/
98 innodb_data_file_path = ibdata1:1G;ibdata2:1G:autoextend
99
100 innodb_log_group_home_dir = /usr/local/mysql/arch
101 innodb_log_files_in_group = 4
102 innodb_log_file_size = 1G
103 innodb_log_buffer_size = 200M
104
105 #根据生产需要，调整pool size
106 innodb_buffer_pool_size = 2G
107 #innodb_additional_mem_pool_size = 50M #deprecated in 5.6
108 tmpdir = /usr/local/mysql/tmp
109
110 innodb_lock_wait_timeout = 1000
111 #innodb_thread_cOncurrency= 0
112 innodb_flush_log_at_trx_commit = 2
113
114 innodb_locks_unsafe_for_binlog=1
115
116 #innodb io features: add for mysql5.5.8
117 performance_schema
118 innodb_read_io_threads=4
119 innodb-write-io-threads=4
120 innodb-io-capacity=200
121 #purge threads change default(0) to 1 for purge
122 innodb_purge_threads=1
123 innodb_use_native_aio=on
124
125 #case-sensitive file names and separate tablespace
126 innodb_file_per_table = 1
127 lower_case_table_names=1
128
129 [mysqldump]
130 quick
131 max_allowed_packet = 128M
132
133 [mysql]
134 no-auto-rehash
135 default-character-set=utf8mb4
136
137 [mysqlhotcopy]
138 interactive-timeout
139
140 [myisamchk]
141 key_buffer_size = 256M
142 sort_buffer_size = 256M
143 read_buffer = 2M
144 write_buffer = 2M
145 [root@hadoop001 local]# groupadd -g 101 dba
146 [root@hadoop001 local]# useradd -u 514 -g dba -G root -d /usr/local/mysql mysqladmin
147 [root@hadoop001 local]# id mysqladmin
148 uid=514(mysqladmin) gid=101(dba) groups=101(dba),0(root)
149 [root@hadoop001 local]# usermod -u 514 -g dba -G root -d /usr/local/mysql mysqladmin
150 [root@hadoop001 local]# cp /etc/skel/.* /usr/local/mysql
151 cp: omitting directory ‘/etc/skel/.’
152 cp: omitting directory ‘/etc/skel/..’
153 [root@hadoop001 local]# vi mysql/.bash_profile
154 [root@hadoop001 local]# cat mysql/.bash_profile
155 # .bash_profile
156 # Get the aliases and functions
157
158 if [ -f ~/.bashrc ]; then
159 . ~/.bashrc
160 fi
161
162 # User specific environment and startup programs
163 export MYSQL_BASE=/usr/local/mysql
164 export PATH=${MYSQL_BASE}/bin:$PATH
165
166
167 unset USERNAME
168
169 #stty erase ^H
170 set umask to 022
171 umask 022
172 PS1=`uname -n`":"'$USER'":"'$PWD'":>"; export PS1
173 [root@hadoop001 local]# chown mysqladmin:dba /etc/my.cnf
174 [root@hadoop001 local]# chmod 640 /etc/my.cnf
175 [root@hadoop001 local]# chown -R mysqladmin:dba /usr/local/mysql
176 [root@hadoop001 local]# chmod -R 755 /usr/local/mysql
177 [root@hadoop001 local]# cd /usr/local/mysql
178 [root@hadoop001 mysql]# cp support-files/mysql.server /etc/rc.d/init.d/mysql
179 [root@hadoop001 mysql]# chmod +x /etc/rc.d/init.d/mysql
180 [root@hadoop001 mysql]# chkconfig --del mysql
181 [root@hadoop001 mysql]# chkconfig --add mysql
182 [root@hadoop001 mysql]# chkconfig --level 345 mysql on
183 [root@hadoop001 mysql]# yum -y install libaio
184 Loaded plugins: fastestmirror
185 Loading mirror speeds from cached hostfile
186 Package libaio-0.3.109-13.el7.x86_64 already installed and latest version
187 Nothing to do
188 [root@hadoop001 mysql]# su - mysqladmin
189 Last login: Sat Jun 29 16:27:28 CST 2019 on pts/0
190 hadoop001:mysqladmin:/usr/local/mysql:>bin/mysqld
191 > --defaults-file=/etc/my.cnf
192 > --user=mysqladmin
193 > --basedir=/usr/local/mysql/
194 > --datadir=/usr/local/mysql/data/
195 > --initialize
196
197 hadoop001:mysqladmin:/usr/local/mysql:>
198 hadoop001:mysqladmin:/usr/local/mysql:>cd data/
199 hadoop001:mysqladmin:/usr/local/mysql/data:>cat hostname.err |grep password
200 2019-06-29T08:54:08.196748Z 1 [Note] A temporary password is generated for root@localhost: FrU?l/Wll2er
201 hadoop001:mysqladmin:/usr/local/mysql/data:>/usr/local/mysql/bin/mysqld_safe --defaults-file=/etc/my.cnf &
202 [1] 13007
203 hadoop001:mysqladmin:/usr/local/mysql/data:>2019-06-29T08:54:49.433242Z mysqld_safe Logging to '/usr/local/mysql/data/hostname.err'.
204 2019-06-29T08:54:49.474207Z mysqld_safe Starting mysqld daemon with databases from /usr/local/mysql/data
205
206 hadoop001:mysqladmin:/usr/local/mysql/data:>mysql -uroot -p'FrU?l/Wll2er'
207 mysql: [Warning] Using a password on the command line interface can be insecure.
208 Welcome to the MySQL monitor. Commands end with ; or g.
209 Your MySQL connection id is 2
210 Server version: 5.7.11-log
211
212 Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
213
214 Oracle is a registered trademark of Oracle Corporation and/or its
215 affiliates. Other names may be trademarks of their respective
216 owners.
217
218 Type 'help;' or 'h' for help. Type 'c' to clear the current input statement.
219
220 mysql> alter user root@localhost identified by 'cdh123';
221 Query OK, 0 rows affected (0.00 sec)
222
223 mysql> GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'cdh123' ;
224 Query OK, 0 rows affected, 1 warning (0.00 sec)
225
226 mysql>
227 mysql>
228 mysql> create database cmf default character set utf8;
229 Query OK, 1 row affected (0.00 sec)
230
231 mysql> GRANT ALL PRIVILEGES ON cmf.* TO 'cmf'@'%' IDENTIFIED BY 'cdh123';
232 Query OK, 0 rows affected, 1 warning (0.00 sec)
233
234 mysql>
235 mysql> create database amon default character set utf8;
236 Query OK, 1 row affected (0.00 sec)
237
238 mysql> GRANT ALL PRIVILEGES ON amon.* TO 'amon'@'%' IDENTIFIED BY 'cdh123';
239 Query OK, 0 rows affected, 1 warning (0.00 sec)
240
241 mysql>
242 mysql> create database hive default character set utf8;
243 Query OK, 1 row affected (0.00 sec)
244
245 mysql> GRANT ALL PRIVILEGES ON hive.* TO 'hive'@'%' IDENTIFIED BY 'cdh123';
246 Query OK, 0 rows affected, 1 warning (0.00 sec)
247
248 mysql>
249 mysql> create database oozie default character set utf8;
250 Query OK, 1 row affected (0.00 sec)
251
252 mysql> GRANT ALL PRIVILEGES ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'cdh123';
253 Query OK, 0 rows affected, 1 warning (0.00 sec)
254
255 mysql>
256 mysql> create database hue default character set utf8;
257 Query OK, 1 row affected (0.00 sec)
258
259 mysql> GRANT ALL PRIVILEGES ON hue.* TO 'hue'@'%' IDENTIFIED BY 'cdh123';
260 Query OK, 0 rows affected, 1 warning (0.00 sec)
261
262 mysql> flush privileges;
263 Query OK, 0 rows affected (0.01 sec)
264
265 mysql> show databases;
266 +--------------------+
267 | Database |
268 +--------------------+
269 | information_schema |
270 | amon |
271 | cmf |
272 | hive |
273 | hue |
274 | mysql |
275 | oozie |
276 | performance_schema |
277 | sys |
278 +--------------------+
279 9 rows in set (0.00 sec)
280
281 mysql> ^C
282 mysql>
283 [2]+ Stopped mysql -uroot -p'FrU?l/Wll2er'
284 hadoop001:mysqladmin:/usr/local/mysql/data:>service mysql restart
285 Shutting down MySQL....2019-06-29T08:56:38.980522Z mysqld_safe mysqld from pid file /usr/local/mysql/data/hostname.pid ended
286 [ OK ]
287 Starting MySQL..[ OK ]
288 [1]- Done /usr/local/mysql/bin/mysqld_safe --defaults-file=/etc/my.cnf
289 hadoop001:mysqladmin:/usr/local/mysql/data:>