集群消化

发表于 | 分类于

##六、安装hadoop

####2.6、hadoop-env.sh的配置

1
2
3
cd /home/hadoop/hadoop_installs/hadoop-2.3.0/etc/hadoop/

vim hadoop-env.sh

1
2
# The java implementation to use.
export JAVA_HOME=/home/hadoop/hadoop_installs/hadoop-2.3.0/jdk1.6.0_26

####2.7、修改core-site.xml

1
2
3
cd /home/hadoop/hadoop_installs/hadoop-2.3.0/etc/hadoop/

vim core-site.xml

1
2
3
4
5
6
7
8
9
<configuration>
<property>
  <name>hadoop.tmp.dir</name>
  <value>file:///data/hadoop-data/tmp</value>
</property>
<property>
 <name>fs.defaultFS</name>
  <value>hdfs://manage2:54300</value>
</property>

####2.9、修改yarn-site.xml

1
2
3
cd /home/hadoop/hadoop_installs/hadoop-2.3.0/etc/hadoop/

vim yarn-site.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
<configuration>
<property>
        <name>yarn.resourcemanager.address</name>
        <value>manage2:8290</value>
</property>
<property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>manage2:8291</value>
</property>
<property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>manage2:8292</value>
</property>
<property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>manage2:8293</value>
</property>
<property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>0.0.0.0:50030</value>
</property>
<property>
        <name>yarn.nodemanager.resource.memory-mb</name>
        <value>10000</value>
</property>
<property>
         <name>yarn.nodemanager.aux-services</name>
         <value>mapreduce_shuffle</value>
</property>
<property>
         <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
         <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
         <name>yarn.nodemanager.resource.cpu-vcores</name>
         <value>16</value>
</property>
<property>
        <name>yarn.nodemanager.local-dirs</name>
        <value>${hadoop.tmp.dir}/nm-local-dir</value>
</property>
<property>
    <name>yarn.log-aggregation-enable</name>
        <value>true</value>
</property>
</configuration>

####2.10、修改hdfs-site.xml

1
sudo gedit /usr/local/hadoop/etc/hadoop/hdfs-site.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
<configuration>  
<property>  
        <name>dfs.replication</name>  
        <value>1</value>  
    </property>  
    <property>  
        <name>dfs.namenode.name.dir</name>  
        <value>file:/usr/local/hadoop/dfs/name</value>  
    </property>  
    <property>  
        <name>dfs.datanode.data.dir</name>  
        <value>file:/usr/local/hadoop/dfs/data</value>  
    </property>  
    <property>                 
            <name>dfs.permissions</name>  
            <value>false</value>  
     </property>  
 </configuration>

####2.11、修改master与slaves

1
2
3
4
5
sudo gedit /usr/local/hadoop/etc/hadoop/masters 

sudo gedit /usr/local/hadoop/etc/hadoop/slaves

添加：localhost

##七、运行hadoop

###①首先在 Hadoop 目录下创建所需的临时目录

1
2
cd /usr/local/hadoop  
mkdir tmp dfs dfs/name dfs/data

###②格式化hdfs文件系统，初次运行

1
2
cd /usr/local/hadoop/
bin/hdfs namenode -format

###③最后开启服务

1
2
sbin/start-dfs.sh  
sbin/start-yarn.sh

##八、状态查看

1
2
3
Namenode information:
http://localhost:50070
来查看Hadoop的信息。

1
2
All Applications：
http://localhost:8088/，