标签：运行   lan   output   yar   new   iterable   env   ufw   AC   

1.  设置网络IP

sudo nmtui

sudo /etc/init.d/networking restart

2.  设置主机名

sudo hostnamectl set-hostname=ubuntuhadoop.smartmap.com

sudo gedit /etc/hosts

127.0.0.1       localhost

# 127.0.0.1     ubuntuhadoop.smartmap.com

# 127.0.0.1     ubuntuhadoop

192.168.1.60    ubuntuhadoop

192.168.1.60    ubuntuhadoop.smartmap.com

# The following lines are desirable for IPv6 capable hosts

::1     ip6-localhost ip6-loopback

fe00::0 ip6-localnet

ff00::0 ip6-mcastprefix

ff02::1 ip6-allnodes

ff02::2 ip6-allrouters

3.  关闭防火墙

sudo ufw disable

4.  安装VMwareTool

https://www.linuxidc.com/Linux/2016-04/130807.htm

5.  安装SSH

5.1.  安装SSH服务

sudo apt-get install -y openssh-server

5.2.  设置SSH

sudo vi /etc/ssh/sshd_config

PermitRootLogin yes

5.3.  启动SSH服务

sudo service ssh start

sudo service ssh restart

5.4.  节点间无密码互访

5.4.1.  zyx用户

cd ~

ssh-keygen -t rsa

cp .ssh/id_rsa.pub .ssh/authorized_keys

5.4.2.  root用户

sudo su – root

cd ~

ssh-keygen -t rsa

cp .ssh/id_rsa.pub .ssh/authorized_keys

exit

6.  设置vim（Ubuntu中的vim太难用了）

sudo gedit /etc/vim/vimrc.tiny

" set compatible

set nocompatible

set backspace=2

" vim: set ft=vim:

7.  安装JDK

7.1.  加入Oracle的JDK仓库

sudo add-apt-repository ppa:webupd8team/java

7.2.  更新

sudo apt-get update

7.3.  安装

sudo apt-get install oracle-java8-installer

注意：java默认安装在 /usr/lib/jvm文件夹下

7.4.  环境变量配置

sudo gedit /etc/profile

export JAVA_HOME=/usr/lib/jvm/java-8-oracle

export JRE_HOME=$JAVA_HOME/jre

export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JRE_HOME/lib

export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.5

export HADOOP_MAPRED_HOME=$HADOOP_HOME

export HADOOP_COMMON_HOME=$HADOOP_HOME

export HADOOP_HDFS_HOME=$HADOOP_HOME

export YARN_HOME=$HADOOP_HOME

export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop

export PATH=$PATH:$HADOOP_HOME/sbin

export PATH=$PATH:$HADOOP_HOME/bin

export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:/usr/local/lib:$HADOOP_HOME/lib/native

export JAVA_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH

souce /etc/profile

8.  安装Hadoop

8.1.  解压hadoop

sudo mkdir /usr/local/hadoop

sudo tar zxvf hadoop-2.7.5.tar.gz -C /usr/local/hadoop

8.2.  创建目录

mkdir /home/zyx/tmp

mkdir /home/zyx/dfs

mkdir /home/zyx/dfs/name

mkdir /home/zyx/dfs/data

mkdir /home/zyx/dfs/checkpoint

mkdir /home/zyx/yarn

mkdir /home/zyx/yarn/local

8.3.  修改Hadoop的配置文件

各配置文件在/usr/local/hadoop/hadoop-2.7.5/etc/hadoop/目录下

8.3.1.  hadoop-env.sh

# The java implementation to use.

# export JAVA_HOME=${JAVA_HOME}

export JAVA_HOME=/usr/lib/jvm/java-8-oracle

export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.5

export PATH=$PATH:/usr/local/hadoop/hadoop-2.7.5/bin

8.3.2.  yarn-env.sh

# some Java parameters

# export JAVA_HOME=/home/y/libexec/jdk1.6.0/

export JAVA_HOME=/usr/lib/jvm/java-8-oracle

8.3.3.  core-site.xml

<configuration>

   <property>

        <name>fs.default.name</name>

        <value>hdfs://192.168.1.60:9000</value>

   </property>

   <property>

        <name>hadoop.tmp.dir</name>

        <value>/home/zyx/tmp</value>

   </property>

</configuration>

8.3.4.  hdfs-site.xml

<configuration>

    <property>

        <name>dfs.namenode.http-address</name>

        <value>192.168.1.60:50070</value>

   </property>

   <property>

        <name>dfs.namenode.http-bind-host</name>

        <value>192.168.1.60</value>

   </property>

   <property>

        <name>dfs.namenode.secondary.http-address</name>

        <value>192.168.1.60:50090</value>

   </property>

   <property>

        <name>dfs.namenode.name.dir</name>

        <value>/home/zyx/dfs/name</value>

   </property>

   <property>

        <name>dfs.datanode.data.dir</name>

        <value>/home/zyx/dfs/data</value>

   </property>

   <property>

        <name>dfs.replication</name>

        <value>1</value>

   </property>

   <property>

        <name>dfs.namenode.checkpoint.dir</name>

        <value>/home/zyx/dfs/checkpoint</value>

   </property>

</configuration>

8.3.5.  mapred-site.xml

sudo cp /usr/local/hadoop/hadoop-2.7.5/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/hadoop-2.7.5/etc/hadoop/mapred-site.xml

<configuration>

   <property>

        <name>mapreduce.framework.name</name>

        <value>yarn</value>

   </property>

</configuration>

8.3.6.  yarn-site.xml

<configuration>

   <property>

        <name>yarn.resourcemanager.hostname</name>

        <value>192.168.1.60</value>

   </property>

   <property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

   </property>

   <property>

        <name>yarn.nodemanager.local-dirs</name>

        <value>/home/zyx/yarn/local</value>

   </property>

</configuration>

8.3.7.  slave

sudo gedit /usr/local/hadoop/hadoop-2.7.5/etc/hadoop/slaves

192.168.1.60

9.  Hadoop启动

9.1.  HDFS文件的格式化

注意在zyx用户下

hadoop namenode -format

9.2.  DFS服务的启动与关停

注意在root用户下

cd /usr/local/hadoop/hadoop-2.7.5/sbin/

sudo ./start-dfs.sh

sudo ./stop-dfs.sh

9.3.  查看数据节点信息

hadoop dfsadmin -report

9.4.  YARN服务的启动与关停

注意在root用户下

cd /usr/local/hadoop/hadoop-2.7.5/sbin/

sudo ./start-yarn.sh

sudo ./stop-yarn.sh

9.5.  查看本机上已启动的服务

sudo jps

9.6.  HDFS的Web UI管理界面

http://192.168.1.60:50070/dfshealth.html#tab-overview

9.7.  MapReduce应用的Web UI管理界面

http://192.168.1.60:8088/cluster

10.       安装Eclipse

10.1.    解压eclipse

sudo mv /home/zyx/software/eclipse-java-neon-3-linux-gtk-x86_64.tar.gz /opt/

sudo tar zxvf /opt/eclipse-java-neon-3-linux-gtk-x86_64.tar.gz

sudo mkdir /opt/eclipse/workspace

10.2.    安装插件

sudo mv /home/zyx/software/hadoop-eclipse-plugin-2.7.3.jar /opt/

sudo cp /opt/hadoop-eclipse-plugin-2.7.3.jar /opt/eclipse/dropins/

ls -la dropins/

10.3.    启动Eclipse

cd /opt/eclipse/

sudo ./eclipse

11.       准备数据

11.1.    查找应用数据

sudo cp /usr/local/hadoop/hadoop-2.7.5/NOTICE.txt /home/zyx/test.txt

sudo chown zyx:zyx /home/zyx/test.txt

11.2.    将数据放入HDFS中

hadoop fs -mkdir /input

hadoop fs -put /home/zyx/test.txt /input

hadoop fs -chmod -R 777 /input/test.txt

12.        配置Eclipse中的Hadoop设置

12.1.    设置Hadoop安装路径

12.2.    设Eclipse的Hadoop视图

12.3.    连接HDFS

12.4.    新建MapReduce工程

12.5.    新建运行的类

WordCount类中的内容

package org.apache.hadoop.examples;

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {

         public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {

                  private final static IntWritable one = new IntWritable(1);

                  private Text word = new Text();

                  public void map(Object key, Text value, Context context) throws IOException, InterruptedException {

                          StringTokenizer itr = new StringTokenizer(value.toString());

                          while (itr.hasMoreTokens()) {

                                   word.set(itr.nextToken());

                                   context.write(word, one);

                          }

                  }

         }

         public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

                  private IntWritable result = new IntWritable();

                  public void reduce(Text key, Iterable<IntWritable> values, Context context)

                                   throws IOException, InterruptedException {

                          int sum = 0;

                          for (IntWritable val : values) {

                                   sum += val.get();

                          }

                          result.set(sum);

                          context.write(key, result);

                  }

         }

         public static void main(String[] args) throws Exception {

                  Configuration conf = new Configuration();

                  Job job = Job.getInstance(conf, "word count");

                  job.setJarByClass(WordCount.class);

                  job.setMapperClass(TokenizerMapper.class);

                  job.setCombinerClass(IntSumReducer.class);

                  job.setReducerClass(IntSumReducer.class);

                  job.setOutputKeyClass(Text.class);

                  job.setOutputValueClass(IntWritable.class);

                  FileInputFormat.addInputPath(job, new Path(args[0]));

                  FileOutputFormat.setOutputPath(job, new Path(args[1]));

                  System.exit(job.waitForCompletion(true) ? 0 : 1);

         }

}

12.6.    新建日志配置文件log4j.properties

在src/目录下新建日志配置文件log4j.properties

其内容如下：

log4j.rootLogger=debug, stdout, R

#log4j.rootLogger=stdout, R  

log4j.appender.stdout=org.apache.log4j.ConsoleAppender 

log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 

#log4j.appender.stdout.layout.ConversionPattern=%5p - %m%n  

log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n 

log4j.appender.R=org.apache.log4j.RollingFileAppender 

log4j.appender.R.File=log4j.log  

log4j.appender.R.MaxFileSize=100KB  

log4j.appender.R.MaxBackupIndex=1  

log4j.appender.R.layout=org.apache.log4j.PatternLayout 

#log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n  

log4j.appender.R.layout.ConversionPattern=%d %p [%c] - %m%n 

#log4j.logger.com.codefutures=DEBUG

12.7.    运行

12.7.1.         运行环境设置

12.7.1.1.      设置输入与输出

hdfs://192.168.1.60:9000/input/test.txt

hdfs://192.168.1.60:9000/output

12.7.1.2.      设置环境变量

HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.5

12.8.    启动运行

12.9.    查看运行结果

Ubuntu-16.04-Desktop +Hadoop2.7.5+Eclipse-Neon的云计算开发环境的搭建(伪分布式方式)

标签：运行   lan   output   yar   new   iterable   env   ufw   AC   

原文地址：https://www.cnblogs.com/gispathfinder/p/9064248.html