(1)机器免登录
ssh-keygen -t rsa
cd ~/.ssh/
cat id_rsa.pub >>
authorized_keys
chmod 600
authorized_keys
[tip].ssh目录的权限必须是700
[tip].ssh/authorized_keys文件权限必须是600
(2)修改配置
cp
mapred-site.xml.template mapred-site.xml
vi mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
vi core-site.xml
<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
</property>
vi yarn-site.xml
<property>
<!--注意这里是中划线-->
<name>yarn.nodemanager.aux-services</name>
<!--注意这里是下划线,否则会报名称不符合规范-->
<value>mapreduce_shuffle</value>
</property>
vi
hdfs-site.xml
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hadoop/data/yarn/hdfs/namenode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hadoop/data/yarn/hdfs/datanode</value>
</property>
vi ~/.bashrc
#Hadoop2
export
HADOOP_HOME=/home/hadoop/soft/hadoop-2.4.0
export
HADOOP_MAPRED_HOME=$HADOOP_HOME
export
HADOOP_COMMON_HOME=$HADOOP_HOME
export
HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export
HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export
PATH=$PATH:$HADOOP_HOME/bin
source ~/.bashrc
(3)启动
hadoop
namenode -format
sh
sbin/start-all.sh
[查看节点]
http://localhost:50070/
[查看任务]
http://localhost:8088/
(4)运行示例
在示例的源码包里有一个类叫
hadoop-2.4.0-src/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/ExampleDriver.java
它是程序的入口
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 |
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.examples; import org.apache.hadoop.examples.dancing.DistributedPentomino; import org.apache.hadoop.examples.dancing.Sudoku; import org.apache.hadoop.examples.pi.DistBbp; import org.apache.hadoop.examples.terasort.TeraGen; import org.apache.hadoop.examples.terasort.TeraSort; import org.apache.hadoop.examples.terasort.TeraValidate; import org.apache.hadoop.util.ProgramDriver; /** * A description of an example program based on its class and a * human-readable description. */ public
class ExampleDriver { public
static void main(String argv[]){ int
exitCode = -1; ProgramDriver pgd = new
ProgramDriver(); try
{ pgd.addClass( "wordcount" , WordCount. class , "A map/reduce program that counts the words in the input files." ); pgd.addClass( "wordmean" , WordMean. class , "A map/reduce program that counts the average length of the words in the input files." ); pgd.addClass( "wordmedian" , WordMedian. class , "A map/reduce program that counts the median length of the words in the input files." ); pgd.addClass( "wordstandarddeviation" , WordStandardDeviation. class , "A map/reduce program that counts the standard deviation of the length of the words in the input files." ); pgd.addClass( "aggregatewordcount" , AggregateWordCount. class , "An Aggregate based map/reduce program that counts the words in the input files." ); pgd.addClass( "aggregatewordhist" , AggregateWordHistogram. class , "An Aggregate based map/reduce program that computes the histogram of the words in the input files." ); pgd.addClass( "grep" , Grep. class , "A map/reduce program that counts the matches of a regex in the input." ); pgd.addClass( "randomwriter" , RandomWriter. class , "A map/reduce program that writes 10GB of random data per node." ); pgd.addClass( "randomtextwriter" , RandomTextWriter. class , "A map/reduce program that writes 10GB of random textual data per node." ); pgd.addClass( "sort" , Sort. class , "A map/reduce program that sorts the data written by the random writer." ); pgd.addClass( "pi" , QuasiMonteCarlo. class , QuasiMonteCarlo.DESCRIPTION); pgd.addClass( "bbp" , BaileyBorweinPlouffe. class , BaileyBorweinPlouffe.DESCRIPTION); pgd.addClass( "distbbp" , DistBbp. class , DistBbp.DESCRIPTION); pgd.addClass( "pentomino" , DistributedPentomino. class , "A map/reduce tile laying program to find solutions to pentomino problems." ); pgd.addClass( "secondarysort" , SecondarySort. class , "An example defining a secondary sort to the reduce." ); pgd.addClass( "sudoku" , Sudoku. class , "A sudoku solver." ); pgd.addClass( "join" , Join. class , "A job that effects a join over sorted, equally partitioned datasets" ); pgd.addClass( "multifilewc" , MultiFileWordCount. class , "A job that counts words from several files." ); pgd.addClass( "dbcount" , DBCountPageView. class , "An example job that count the pageview counts from a database." ); pgd.addClass( "teragen" , TeraGen. class , "Generate data for the terasort" ); pgd.addClass( "terasort" , TeraSort. class , "Run the terasort" ); pgd.addClass( "teravalidate" , TeraValidate. class , "Checking results of terasort" ); exitCode = pgd.run(argv); } catch (Throwable e){ e.printStackTrace(); } System.exit(exitCode); } } |
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.4.0.jar pi 5 10
[默认配置的话运行输出会有这么一句]
running in uber mode : false
运行分本地模式 uber mode 和 no-uber mode
uber mode对于小作业共享container -
MapTask,ReduceTask会使用MRAppMaster所在的container
hadoop2.4-single,布布扣,bubuko.com
原文地址:http://www.cnblogs.com/really-dt/p/3774713.html