Hadoop2.6+Hive1.1.1+Sqoop1.4.6安装


(1)服务器准备
192.168.20.130 hadoop-master1 # NameNode,ResourceManager
192.168.20.131 hadoop-slave1    # SecondaryNameNode,NodeManager,DataNode
192.168.20.132 hadoop-slave2    # NodeManager,DataNode
192.168.20.133 hadoop-slave3    # NodeManager,DataNode


(2)安装java，及ssh免密配置


(3)新建文件夹
/root/hadoop/data/hdfs/data
/root/hadoop/data/hdfs/name
/root/hadoop/data/hdfs/tmp


(4)解压hadoop,hive,sqoop
/root/hadoop/hadoop-2.6.0
/root/hive/apache-hive-1.1.1-bin
/root/sqoop/sqoop-1.4.6


(4-1)安装hadoop
vi /root/hadoop/hadoop-2.6.0/etc/hadoop/yarn-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_171


vi /root/hadoop/hadoop-2.6.0/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_171


vi /root/hadoop/hadoop-2.6.0/etc/hadoop/core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
 <name>hadoop.tmp.dir</name>
 <value>/root/hadoop/data/hdfs/tmp</value>
 <description>A base for other temporary directories.</description>
</property>
<property>
 <name>io.file.buffer.size</name>
 <value>131072</value>
</property>
<property>
 <name>fs.default.name</name>
 <value>hdfs://hadoop-master1:9000</value>
</property>
<property>
 <name>hadoop.proxyuser.root.hosts</name>
 <value>*</value>
</property>
<property>
 <name>hadoop.proxyuser.root.groups</name>
 <value>*</value>
</property>
<property>
 <name>fs.checkpoint.period</name>
 <value>60</value>
 <description>The number of seconds between two periodic checkpoints.</description>
</property>
<property>
 <name>fs.checkpoint.size</name>
 <value>67108864</value>
</property>
</configuration>


vi /root/hadoop/hadoop-2.6.0/etc/hadoop/hdfs-site.xml


<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
 <name>dfs.replication</name>
 <value>3</value>
</property>
<property>
 <name>dfs.namenode.name.dir</name>
 <value>/root/hadoop/data/hdfs/name</value>
 <final>true</final>
</property>
<property>
 <name>dfs.datanode.data.dir</name>
 <value>/root/hadoop/data/hdfs/data</value>
 <final>true</final>
</property>
<property>
 <name>dfs.http.address</name>
 <value>hadoop-master1:50070</value>
</property>
<property>
 <name>dfs.namenode.secondary.http-address</name>
 <value>hadoop-slave1:50090</value>
</property>
<property>
 <name>dfs.webhdfs.enabled</name>
 <value>true</value>
</property>
<property>
 <name>dfs.permissions</name>
 <value>false</value>
</property>
</configuration>


vi /root/hadoop/hadoop-2.6.0/etc/hadoop/mapred-site.xml


<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
<property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
</property>
</configuration>

vi /root/hadoop/hadoop-2.6.0/etc/hadoop/yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
<property>
 <name>yarn.resourcemanager.address</name>
 <value>hadoop-master1:18040</value>
</property>
<property>
 <name>yarn.resourcemanager.scheduler.address</name>
 <value>hadoop-master1:18030</value>
</property>
<property>
 <name>yarn.resourcemanager.webapp.address</name>
 <value>hadoop-master1:18088</value>
</property>
<property>
 <name>yarn.resourcemanager.resource-tracker.address</name>
 <value>hadoop-master1:18025</value>
</property>
<property>
 <name>yarn.resourcemanager.admin.address</name>
 <value>hadoop-master1:18141</value>
</property>
<property>
 <name>yarn.nodemanager.aux-services</name>
 <value>mapreduce_shuffle</value>
</property>
<property>
 <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
 <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
</configuration>


vi /root/hadoop/hadoop-2.6.0/etc/hadoop/masters
hadoop-master1


vi /root/hadoop/hadoop-2.6.0/etc/hadoop/slaves
hadoop-slave1
hadoop-slave2
hadoop-slave3


运行hadoop,yarn
/root/hadoop/hadoop-2.6.0/bin/hadoop namenode -format
/root/hadoop/hadoop-2.6.0/sbin/start-all.sh
/root/hadoop/hadoop-2.6.0/sbin/stop-all.sh
http://192.168.30.130:50070/dfshealth.html
http://192.168.30.130:18088/cluster
/root/hadoop/hadoop-2.6.0/bin/hadoop dfsadmin -report


(4-2)安装hive


vi /root/hive/apache-hive-1.1.1-bin/conf/hive-env.sh

HADOOP_HOME=/root/hadoop/hadoop-2.6.0


vi /root/hive/apache-hive-1.1.1-bin/conf/hive-exec-log4j.properties
hive.log.dir=/root/hive/apache-hive-1.1.1-bin/logs


vi /root/hive/apache-hive-1.1.1-bin/conf/hive-log4j.properties
hive.log.dir=/root/hive/apache-hive-1.1.1-bin/logs


vi /root/hive/apache-hive-1.1.1-bin/conf/hive-site.xml
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
 <!--Hive作业的HDFS根目录位置 -->
 <property>
  <name>hive.exec.scratchdir</name>
  <value>/user/hive/tmp</value>
 </property>
 <!--Hive作业的HDFS根目录创建写权限 -->
 <property>
  <name>hive.scratch.dir.permission</name>
  <value>755</value>
 </property>
 <!--hdfs上hive元数据存放位置 -->
 <property>
   <name>hive.metastore.warehouse.dir</name>
   <value>/user/hive/warehouse</value>
 </property>
 <!--连接数据库地址，名称 -->
 <property>
   <name>javax.jdo.option.ConnectionURL</name>
   <value>jdbc:mysql://hadoop-master1:3306/hive_metastore?createDatabaseIfNotExist=true</value>
 </property>
 <!--连接数据库驱动 -->
 <property>
   <name>javax.jdo.option.ConnectionDriverName</name>
   <value>com.mysql.jdbc.Driver</value>
 </property>
 <!--连接数据库用户名称 -->
 <property>
   <name>javax.jdo.option.ConnectionUserName</name>
   <value>root</value>
 </property>
 <!--连接数据库用户密码 -->
 <property>
   <name>javax.jdo.option.ConnectionPassword</name>
   <value>123456</value>
 </property>
 <!--客户端显示当前查询表的头信息 -->
  <property>
   <name>hive.cli.print.header</name>
   <value>true</value>
 </property>
 <!--客户端显示当前数据库名称信息 -->
 <property>
   <name>hive.cli.print.current.db</name>
   <value>true</value>
 </property>
</configuration>


cp /root/hive/apache-hive-1.1.1-bin/lib/jline-2.12.jar /root/hadoop/hadoop-2.6.0/share/hadoop/yarn/lib/
上传 mysql-connector-java-5.1.26.jar  到 /root/hive/apache-hive-1.1.1-bin/lib
启动相应服务 /root/hive/apache-hive-1.1.1-bin/bin


(4-3)安装sqoop


vi /root/sqoop/sqoop-1.4.6/conf/sqoop-env.sh
export HADOOP_COMMON_HOME=/root/hadoop/hadoop-2.6.0
export HADOOP_MAPRED_HOME=/root/hadoop/hadoop-2.6.0
export HIVE_HOME=/root/hive/apache-hive-1.1.1-bin


上传 mysql-connector-java-5.1.26.jar  到 /root/sqoop/sqoop-1.4.6/lib
上传 sqoop-1.4.6.jar 到 /root/sqoop/sqoop-1.4.6/lib
测试连接 ./sqoop  list-databases --connect jdbc:mysql://127.0.0.1:3306/ --username root --password 123456