搭建Hadoop Yarn环境 (ARM)

Aug 15 2020

ARM上搭建Hadoop Yarn的指导。

1. yum update -y
   yum install -y \
       autoconf \
       automake \
       libtool \
       cmake \
       pkg-config \
       curl \
       sudo \
       git \
       hostname \
       bind-utils \
       openssh-server \
       snappy \
       snappy-devel \
       bzip2 \
       libzip-devel \
       java-1.8.0-openjdk \
       java-1.8.0-openjdk-devel
   

2. 新增Hadoop用户

   groupadd hadoop
   useradd -m -d /home/hadoop -s /bin/bash hadoop -g hadoop
   echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
   

3. curl -L https://downloads.apache.org/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-aarch64.tar.gz | tar zx
   

4. 环境变量准备

   export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.265.b01-6.oe1.aarch64
   export HADOOP_HOME=/home/hadoop/hadoop-3.3.0
   

5. 配置添加
   cd $HADOOP_HOME/etc/hadoop/
   core-site.xml

   <configuration>
       <property>
           <name>fs.defaultFS</name>
           <value>hdfs://localhost:9000</value>
       </property>
   </configuration>
   

hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.name.dir</name>
        <value>/home/hadoop/dfs/name</value>
    </property>
    <property>
        <name>dfs.data.dir</name>
        <value>/home/hadoop/dfs/data</value>
    </property>
</configuration>

mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <!--Deploy Hadoop on K8s, we need to specify a certain port range and expose by k8s Service-->
    <property>
        <name>yarn.app.mapreduce.am.job.client.port-range</name>
        <value>60000-60010</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.webapp.port-range</name>
        <value>60011-60020</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>
    <!--
    <property>
      <name>mapreduce.map.output.compress</name>
      <value>true</value>
    </property>
    <property>
      <name>mapreduce.map.output.compress.codec</name>
      <value>org.apache.hadoop.io.compress.ZStandardCodec</value>
    </property>
    -->
</configuration>

yarn-site.xml

<configuration>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>localhost</value>
    </property>
    <property>
        <name>yarn.nodemanager.address</name>
        <value>0.0.0.0:40555</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
    </property>
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
</configuration>

1. 启动服务
   先格式化hdfs

   hdfs namenode -format
   

启动所有服务

start-all.sh