Teach you how to build a Hadoop 3.x pseudo cluster on Tencent Cloud

$ cd /home/centos
$ mkdir software
$ mkdir module

$ cd software
# Then drag the file in

$ tar -zxvf jdk-8u212-linux-x64.tar.gz -C ../module
$ tar -zxvf hadoop-3.1.3.tar.gz -C ../module

$ cd /etc/profile.d/
$ vim my_env.sh

#JAVA_HOME,PATH 
# export is promoted to a global variable. If your path is different from mine, remember to use your own path here export JAVA_HOME=/home/centos/module/jdk1.8.0_212
export PATH=$PATH:$JAVA_HOME/bin

#HADOOP_HOME
export HADOOP_HOME=/home/centos/module/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin

$ source /etc/profile

$ hadoop version 

$ java 

$ ssh-keygen -t rsa

$ ssh-copy-id local hostname

vi /etc/hosts
#The configuration I keep here is that the master is configured with the intranet of Tencent Cloud. If the external network is configured, the eclipse client will not be able to connect to hadoop
::1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
172.16.0.3 master
127.0.0.1 localhost

vi /etc/sysconfig/network 
#Change HOSTNAME to master
HOSTNAME=master

$ hostnamectl --static set-hostname master

$ systemctl disable firewalld #Permanent

$ cd /home/centos/module/hadoop-3.1.3/etc/hadoop

<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://Tencent Cloud intranet ip address:9820</value>
  </property>
 
  <property>
    <name>hadoop.tmp.dir</name>
    <value>/home/centos/module/hadoop-3.1.3/data/tmp</value>
  </property>
	<!-- Permissions to operate HDFS through the web interface -->
  <property>
    <name>hadoop.http.staticuser.user</name>
    <value>root</value>
  </property>
    <!-- Hive compatibility configuration later -->
  <property>
    <name>hadoop.proxyuser.root.hosts</name>
    <value>*</value>
  </property>
  <property>
    <name>hadoop.proxyuser.root.groups</name>
    <value>*</value>
    </property>
</configuration>

<configuration>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
  <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>Tencent Cloud intranet IP address: 9868</value>
  </property>
</configuration>

export JAVA_HOME=/home/centos/module/jdk1.8.0_212

<configuration>

  <!-- Reducer obtains data -->
  <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <!-- Specify the address of YARN's ResourceManager -->
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>master</value>
  </property>
<!-- Environment variables are inherited from the NodeManagers' container environment properties. For mapreduce applications, in addition to the default value hadoop op_mapred_home should be added. The attribute values ​​are as follows -->
  <property>
    <name>yarn.nodemanager.env-whitelist</name>
 <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
    <!-- Solve the problem that Yarn exceeds the virtual memory limit when executing the program and the Container is killed -->
  <property>
    <name>yarn.nodemanager.pmem-check-enabled</name>
    <value>false</value>
  </property>
  <property>
    <name>yarn.nodemanager.vmem-check-enabled</name>
    <value>false</value>
  </property>
    <!-- Hive compatibility configuration later -->
  <property>
    <name>yarn.scheduler.minimum-allocation-mb</name>
    <value>512</value>
  </property>
  <property>
    <name>yarn.scheduler.maximum-allocation-mb</name>
    <value>4096</value>
  </property>
  <property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>4096</value>
    </property>
  
  <!-- Enable log aggregation-->
  <property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
  </property>
  <!-- Access path -->
  <property> 
    <name>yarn.log.server.url</name> 
    <value>http://172.17.0.13:19888/jobhistory/logs</value>
  </property>
  <!-- Save for 7 days -->
  <property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>604800</value>
  </property>
</configuration>

<!-- History server address-->
<property>
  <name>mapreduce.jobhistory.address</name>
  <value>Tencent Cloud intranet ip:10020</value>
</property>

<!-- History server web address-->
<property>
  <name>mapreduce.jobhistory.webapp.address</name>
  <value>Tencent Cloud intranet ip:19888</value>
</property>

$ hdfs namenode -format

$ start-dfs.sh

[root@VM_0_13_centos hadoop]# jps
20032 Jps
30900 DataNode
31355 SecondaryNameNode
30559 NameNode

$ vim mycluster

#!/bin/bash
case $1 in
"start")
#dfs yarn history
start-dfs.sh
start-yarn.sh
mapred --daemon start historyserver
;;
"stop")
# dfs yarn history
stop-dfs.sh
stop-yarn.sh
mapred --daemon stop historyserver
;;
*)
echo "args is error! please input start or stop"
;;
esac

$ chmod u+x mycluster

$ mycluster start

$ jps
23680 NodeManager
24129 JobHistoryServer
22417 DataNode
24420 Jps
22023 NameNode
23384 ResourceManager
22891 SecondaryNameNode

$ mycluster stop

$ vim /home/centos/module/hadoop-3.1.3/share/hadoop/hdfs/webapps/static/dfs-dust.js

return new Date(Number(v)).toLocaleString();

$ mycluster start

$ mkdir temdatas

$ vim text.txt

$ hdfs dfs -put text.txt /

$ hdfs dfs -get /text.txt ./text1.txt

#Or you can write it in vim and upload it yourself $ hdfs dfs -put wordcount.txt /input

$ hadoop jar /home/centos/module/hadoop-3.1.3/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar wordcount /input /output

#Pull hdfs file [root@master mydata]# hdfs dfs -get /output ./
# View the results [root@master output]# cat part-r-00000 
a 2
b 3
c 2
d 1
e 1
f 1