CentOS 编译 Hadoop

CentOS下手动编译Hadoop-2.9.2

Posted by leone on 2019-03-20

CentOS 下编译 Hadoop

系统环境

1
2
3
4
5
6
7
8
系统: CentOS-7.3_x86_64
protobuf: protoc-2.5.0
maven: maven-3.6.0
hadoop: hadoop-2.9.2
java: jdk1.8.0_131
cmake: cmake-2.8.12.2
OpenSSL: OpenSSL 1.0.2k-fips
findbugs: findbugs-1.3.9

准备工作

1
2
# 安装编译需要的依赖库
yum install -y lzo-devel zlib-devel autoconf automake libtool cmake openssl-devel cmake gcc gcc-c++

安装maven

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# 下载maven
$ wget https://mirrors.shu.edu.cn/apache/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz

# 解压缩
$ tar -zxvf apache-maven-3.6.0-bin.tar.gz -C /usr/local/

# 配置到系统环境变量
$ vim /etc/profile

export MAVEN_HOME=/usr/local/apache-maven-3.6.0
export PATH=$PATH:$MAVEN_HOME/bin

# 刷新配置文件
$ source /etc/profile

# 验证maven安装是是否成功
$ mvn -version

[root@localhost ~]# mvn -version
Apache Maven 3.6.0 (ff8f5e7444045639af65f6095c62210b5713f426; 2018-10-25T03:39:06+08:00)
Maven home: /usr/local/apache-maven-3.6.0
Java version: 1.8.0_131, vendor: Oracle Corporation
Java home: /usr/local/jdk1.8.0_131/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "4.20.13-1.el7.elrepo.x86_64", arch: "amd64", family: "unix"

安装protobuf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# 安装依赖环境
$ yum groupinstall Development tools -y

# 下载
$ https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz

# 解压缩
$ tar -zxvf protobuf-2.5.0.tar.gz

# 进入解压目录 配置安装路径(--prefix=/usr/local/protobuf-2.5.0)
$ ./configure --prefix=/usr/local/protobuf-2.5.0

# 编译
$ make

# 验证编译文件
$ make check

# 安装
$ make install

# 配置protobuf环境变量
$ vim /etc/profile

# 刷新配置文件
$ source /etc/profile

export PROTOCBUF_HOME=/usr/local/protobuf-2.5.0
export PATH=$PATH:$PROTOCBUF_HOME/bin

# 验证是否安装成功
$ protoc --version

[root@localhost ~]# protoc --version
libprotoc 2.5.0

安装findbugs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

# 下载
$ https://jaist.dl.sourceforge.net/project/findbugs/findbugs/1.3.9/findbugs-1.3.9.tar.gz

# 解压缩
$ tar -zxvf findbugs-1.3.9.tar.gz -C /usr/local/findbugs-1.3.9

# 配置系统环境变量
$ vim /etc/profile

export FINDBUGS_HOME=/usr/local/findbugs-1.3.9
export PATH=$PATH:$FINDBUGS_HOME/bin

# 刷新配置文件
$ source /etc/profile

# 验证是否安装成功
$ findbugs -version

[root@localhost ~]# findbugs -version
1.3.9

编译hadoop

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

# 下载hadoop的源码
$ wget https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-2.9.2/hadoop-2.9.2-src.tar.gz

# 解压缩源码
$ tar -zxvf hadoop-2.9.2-src.tar.gz

# 进入hadoop的源码目录开始编译
$ mvn package -Pdist,native -DskipTests -Dtar

# 编译完成

[INFO] Reactor Summary:
[INFO]
[INFO] Apache Hadoop Main ................................. SUCCESS [ 36.760 s]
[INFO] Apache Hadoop Build Tools .......................... SUCCESS [ 22.223 s]
[INFO] Apache Hadoop Project POM .......................... SUCCESS [ 10.227 s]
[INFO] Apache Hadoop Annotations .......................... SUCCESS [ 10.297 s]
[INFO] Apache Hadoop Assemblies ........................... SUCCESS [ 0.291 s]
[INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [ 17.249 s]
[INFO] Apache Hadoop Maven Plugins ........................ SUCCESS [ 16.091 s]
[INFO] Apache Hadoop MiniKDC .............................. SUCCESS [01:55 min]
[INFO] Apache Hadoop Auth ................................. SUCCESS [01:23 min]
[INFO] Apache Hadoop Auth Examples ........................ SUCCESS [ 5.916 s]
[INFO] Apache Hadoop Common ............................... SUCCESS [02:48 min]
[INFO] Apache Hadoop NFS .................................. SUCCESS [ 7.323 s]
[INFO] Apache Hadoop KMS .................................. SUCCESS [ 21.010 s]
[INFO] Apache Hadoop Common Project ....................... SUCCESS [ 0.077 s]
[INFO] Apache Hadoop HDFS Client .......................... SUCCESS [01:05 min]
[INFO] Apache Hadoop HDFS ................................. SUCCESS [01:12 min]
[INFO] Apache Hadoop HDFS Native Client ................... SUCCESS [ 2.842 s]
[INFO] Apache Hadoop HttpFS ............................... SUCCESS [ 24.130 s]
[INFO] Apache Hadoop HDFS BookKeeper Journal .............. SUCCESS [ 16.353 s]
[INFO] Apache Hadoop HDFS-NFS ............................. SUCCESS [ 5.314 s]
[INFO] Apache Hadoop HDFS-RBF ............................. SUCCESS [ 29.657 s]
[INFO] Apache Hadoop HDFS Project ......................... SUCCESS [ 0.063 s]
[INFO] Apache Hadoop YARN ................................. SUCCESS [ 0.075 s]
[INFO] Apache Hadoop YARN API ............................. SUCCESS [ 14.514 s]
[INFO] Apache Hadoop YARN Common .......................... SUCCESS [01:14 min]
[INFO] Apache Hadoop YARN Registry ........................ SUCCESS [ 6.258 s]
[INFO] Apache Hadoop YARN Server .......................... SUCCESS [ 0.072 s]
[INFO] Apache Hadoop YARN Server Common ................... SUCCESS [ 36.000 s]
[INFO] Apache Hadoop YARN NodeManager ..................... SUCCESS [ 14.787 s]
[INFO] Apache Hadoop YARN Web Proxy ....................... SUCCESS [ 4.074 s]
[INFO] Apache Hadoop YARN ApplicationHistoryService ....... SUCCESS [ 19.797 s]
[INFO] Apache Hadoop YARN Timeline Service ................ SUCCESS [ 10.855 s]
[INFO] Apache Hadoop YARN ResourceManager ................. SUCCESS [ 22.955 s]
[INFO] Apache Hadoop YARN Server Tests .................... SUCCESS [ 1.288 s]
[INFO] Apache Hadoop YARN Client .......................... SUCCESS [ 6.973 s]
[INFO] Apache Hadoop YARN SharedCacheManager .............. SUCCESS [ 4.901 s]
[INFO] Apache Hadoop YARN Timeline Plugin Storage ......... SUCCESS [ 4.682 s]
[INFO] Apache Hadoop YARN Router .......................... SUCCESS [ 5.892 s]
[INFO] Apache Hadoop YARN TimelineService HBase Backend ... SUCCESS [01:06 min]
[INFO] Apache Hadoop YARN Timeline Service HBase tests .... SUCCESS [ 16.535 s]
[INFO] Apache Hadoop YARN Applications .................... SUCCESS [ 0.052 s]
[INFO] Apache Hadoop YARN DistributedShell ................ SUCCESS [ 3.429 s]
[INFO] Apache Hadoop YARN Unmanaged Am Launcher ........... SUCCESS [ 2.224 s]
[INFO] Apache Hadoop YARN Site ............................ SUCCESS [ 0.042 s]
[INFO] Apache Hadoop YARN UI .............................. SUCCESS [ 0.052 s]
[INFO] Apache Hadoop YARN Project ......................... SUCCESS [ 7.005 s]
[INFO] Apache Hadoop MapReduce Client ..................... SUCCESS [ 0.213 s]
[INFO] Apache Hadoop MapReduce Core ....................... SUCCESS [ 20.400 s]
[INFO] Apache Hadoop MapReduce Common ..................... SUCCESS [ 18.649 s]
[INFO] Apache Hadoop MapReduce Shuffle .................... SUCCESS [ 4.872 s]
[INFO] Apache Hadoop MapReduce App ........................ SUCCESS [ 9.835 s]
[INFO] Apache Hadoop MapReduce HistoryServer .............. SUCCESS [ 6.539 s]
[INFO] Apache Hadoop MapReduce JobClient .................. SUCCESS [ 5.527 s]
[INFO] Apache Hadoop MapReduce HistoryServer Plugins ...... SUCCESS [ 2.240 s]
[INFO] Apache Hadoop MapReduce Examples ................... SUCCESS [ 6.176 s]
[INFO] Apache Hadoop MapReduce ............................ SUCCESS [ 2.566 s]
[INFO] Apache Hadoop MapReduce Streaming .................. SUCCESS [ 8.532 s]
[INFO] Apache Hadoop Distributed Copy ..................... SUCCESS [ 8.918 s]
[INFO] Apache Hadoop Archives ............................. SUCCESS [ 2.482 s]
[INFO] Apache Hadoop Archive Logs ......................... SUCCESS [ 2.645 s]
[INFO] Apache Hadoop Rumen ................................ SUCCESS [ 6.271 s]
[INFO] Apache Hadoop Gridmix .............................. SUCCESS [ 4.607 s]
[INFO] Apache Hadoop Data Join ............................ SUCCESS [ 3.110 s]
[INFO] Apache Hadoop Ant Tasks ............................ SUCCESS [ 2.631 s]
[INFO] Apache Hadoop Extras ............................... SUCCESS [ 3.344 s]
[INFO] Apache Hadoop Pipes ................................ SUCCESS [ 0.479 s]
[INFO] Apache Hadoop OpenStack support .................... SUCCESS [ 5.677 s]
[INFO] Apache Hadoop Amazon Web Services support .......... SUCCESS [01:57 min]
[INFO] Apache Hadoop Azure support ........................ SUCCESS [ 25.907 s]
[INFO] Apache Hadoop Aliyun OSS support ................... SUCCESS [ 30.414 s]
[INFO] Apache Hadoop Client ............................... SUCCESS [ 7.438 s]
[INFO] Apache Hadoop Mini-Cluster ......................... SUCCESS [ 1.394 s]
[INFO] Apache Hadoop Scheduler Load Simulator ............. SUCCESS [ 8.115 s]
[INFO] Apache Hadoop Resource Estimator Service ........... SUCCESS [ 10.685 s]
[INFO] Apache Hadoop Azure Data Lake support .............. SUCCESS [ 18.561 s]
[INFO] Apache Hadoop Tools Dist ........................... SUCCESS [ 18.124 s]
[INFO] Apache Hadoop Tools ................................ SUCCESS [ 0.245 s]
[INFO] Apache Hadoop Distribution ......................... SUCCESS [01:01 min]
[INFO] Apache Hadoop Cloud Storage ........................ SUCCESS [ 1.359 s]
[INFO] Apache Hadoop Cloud Storage Project ................ SUCCESS [ 0.028 s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 25:06 min
[INFO] Finished at: 2019-03-21T11:54:52+08:00
[INFO] Final Memory: 135M/384M
[INFO] ------------------------------------------------------------------------
[root@localhost hadoop-2.9.2-src]#

# 编译完成的文件在hadoop-dist/target/目录下
$ ll hadoop-dist/target/

[root@localhost hadoop-2.9.2-src]# ll hadoop-dist/target/
total 949892
drwxr-xr-x 2 root root 4096 Mar 21 11:53 antrun
drwxr-xr-x 3 root root 4096 Mar 21 11:53 classes
-rw-r--r-- 1 root root 2209 Mar 21 11:53 dist-layout-stitching.sh
-rw-r--r-- 1 root root 635 Mar 21 11:53 dist-tar-stitching.sh
drwxr-xr-x 9 root root 4096 Mar 21 11:53 hadoop-2.9.2
-rw-r--r-- 1 root root 323710978 Mar 21 11:54 hadoop-2.9.2.tar.gz
-rw-r--r-- 1 root root 31694 Mar 21 11:53 hadoop-dist-2.9.2.jar
-rw-r--r-- 1 root root 648827621 Mar 21 11:54 hadoop-dist-2.9.2-javadoc.jar
-rw-r--r-- 1 root root 29003 Mar 21 11:54 hadoop-dist-2.9.2-sources.jar
-rw-r--r-- 1 root root 29003 Mar 21 11:54 hadoop-dist-2.9.2-test-sources.jar
drwxr-xr-x 2 root root 4096 Mar 21 11:54 javadoc-bundle-options
drwxr-xr-x 2 root root 4096 Mar 21 11:53 maven-archiver
drwxr-xr-x 3 root root 4096 Mar 21 11:53 maven-shared-archive-resources
drwxr-xr-x 3 root root 4096 Mar 21 11:53 test-classes
drwxr-xr-x 2 root root 4096 Mar 21 11:53 test-dir

总结

到此hadoop的编译已经完成 过程中遇到了很多各种各样的问题上述步骤都是经过验证的如在安装过程中遇到问题可以留言,谢谢!