01_部署Hadoop(伪分布式模式)

  $ sudo apt-get install ssh
  $ sudo apt-get install pdsh

tar -xvf  hadoop-3.1.3.tar.gz

#指定jdk路径
export JAVA_HOME=/home/software/jdk1.8
source /etc/hadoop/hadoop-env.sh

<!--指定namenode地址-->
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

<!--指定hdfs保存数据副本的数量，默认为3，伪分布式模式设置为1-->
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

$ bin/hdfs namenode -format

 $ sbin/start-dfs.sh

1.需要在start-dfs.sh和stop-dfs.sh文件中添加下列参数：

#!/usr/bin/env bash
HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

1.创建目录
bin/hdfs dfs -mkdir /user

2.上传文件
bin/hdfs dfs -put 文件名称 /user

$ sbin/stop-dfs.sh

<!--指定mr运行在yarn上 -->
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>
</configuration>

<!--NodeManager获取数据的方式 -->
<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
</configuration>

$ sbin/start-yarn.sh

#对于start-yarn.sh和stop-yarn.sh文件，添加下列参数：

#!/usr/bin/env bash
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root