azkaban是一款工作流任务调度器,类似于crontab的图形化管理,功能性更高于crontab。
相关下载
相关资料
相关软件包及版本信息
mysql(5.6.36):
MySQL-client-5.6.36-1.el7.x86_64.rpm
MySQL-server-5.6.36-1.el7.x86_64.rpm
azkaban(3.20.1):
azkaban-3.20.1.tar.gz
java(1.8.0_112)
ps:这里使用的azkaban都是编译好的。mysql使用的centos 7的 rpm包,java安装参照cdh5安装文档
安装mysql
上传以下2个包到/usr/local/src
MySQL-client-5.6.36-1.el7.x86_64.rpm
MySQL-server-5.6.36-1.el7.x86_64.rpm
安装相关依赖
yum -y install autoconf
卸载mariadb
yum -y remove mariadb-libs-5.5.52-1.el7.x86_64
ps:mariadb是mysql的一个分支,和mysql相冲突
使用rpm安装mysql
rpm -ivh MySQL-client-5.6.36-1.el7.x86_64.rpm
rpm -ivh MySQL-server-5.6.36-1.el7.x86_64.rpm
初始化数据库
mkdir /opt/mysql
chown mysql.mysql /opt/mysql
mysql_install_db --datadir=/opt/mysql --user=mysql
修改启动文件里边的datapath路径
vi /etc/init.d/mysql 47行改为:
datadir=/opt/mysql
相关配置
vi /usr/my.cnf 增加mysqld域里边的内容如下:
#指定内网监听3306
bind-address = 192.168.91.70
#跳过hosts解析
skip-name-resolve
启动与关闭
/etc/init.d/mysql start
/etc/init.d/mysql stop
修改root密码
使用mysql命令直接进入mysql交互式shell:
mysql> set password=password("root");
Query OK, 0 rows affected (0.00 sec)
mysql> flush privileges
-> ;
Query OK, 0 rows affected (0.00 sec)
mysql>
创建azkaban数据库和hadoop用户
使用mysql命令直接进入mysql交互式shell:
mysql> create database azkaban;
Query OK, 1 row affected (0.00 sec)
mysql> grant all on azkaban.* to hadoop@'%' identified by "hadoop";
Query OK, 0 rows affected (0.00 sec)
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
mysql> quit
Bye
安装azkaban
上传azkaban相关包到/usr/local/src
azkaban-3.20.1.tar.gz
创建相关用户和目录
useradd 9euser
mkdir /usr/local/azkaban
解压和复制azkaban
解压和复制相关需要文件
cd /usr/local/src
tar -zxf azkaban-3.20.1.tar.gz
cd tar/
tar -zxf azkaban-exec-server-0.1.0-SNAPSHOT.tar.gz
tar -zxf azkaban-solo-server-0.1.0-SNAPSHOT.tar.gz
tar -zxf azkaban-web-server-0.1.0-SNAPSHOT.tar.gz
cp azkaban-solo-server-0.1.0-SNAPSHOT/conf -r azkaban-exec-server-0.1.0-SNAPSHOT
cp azkaban-solo-server-0.1.0-SNAPSHOT/conf -r azkaban-web-server-0.1.0-SNAPSHOT
cp azkaban-solo-server-0.1.0-SNAPSHOT/plugins -r azkaban-exec-server-0.1.0-SNAPSHOT
cp azkaban-solo-server-0.1.0-SNAPSHOT/plugins -r azkaban-web-server-0.1.0-SNAPSHOT
cp azkaban-solo-server-0.1.0-SNAPSHOT/lib/slf4j-log4j12-1.7.21.jar azkaban-web-server-0.1.0-SNAPSHOT/lib
mv azkaban-web-server-0.1.0-SNAPSHOT /usr/local/azkaban
mv azkaban-exec-server-0.1.0-SNAPSHOT /usr/local/azkaban
cd /usr/local/azkaban
ln -snf azkaban-exec-server-0.1.0-SNAPSHOT azkaban-exec-server
ln -snf azkaban-web-server-0.1.0-SNAPSHOT azkaban-web-server
chown 9euser.9euser -R /usr/local/azkaban/azkaban-web-server
chown 9euser.9euser -R /usr/local/azkaban/azkaban-exec-server
chown 9euser.9euser -R /usr/local/azkaban/azkaban-web-server-0.1.0-SNAPSHOT
chown 9euser.9euser -R /usr/local/azkaban/azkaban-exec-server-0.1.0-SNAPSHOT
执行mysql相关azkaban表
cd /usr/local/src/tar
tar -zxf azkaban-sql-0.1.0-SNAPSHOT.tar.gz
mysql -uhadoop -phadoop -h192.168.91.70 -D azkaban -e "source /usr/local/src/tar/azkaban-sql-0.1.0-SNAPSHOT/create-all-sql-0.1.0-SNAPSHOT.sql;"
配置azkaban
azkaban分2块组成:
1.web 对应目录 /usr/local/azkaban/azkaban-web-server
2.exec 对应目录 /usr/local/azkaban/azkaban-exec-server
azkaban-web-server配置
cd /usr/local/azkaban/azkaban-web-server/conf
主要配置文件azkaban.properties:
# Azkaban Personalization Settings
azkaban.name=9euser
azkaban.label=9euser Azkaban
azkaban.color=#FF3601
azkaban.default.servlet.path=/index
web.resource.dir=web/
default.timezone.id=Asia/Shanghai
# Azkaban UserManager class
user.manager.class=azkaban.user.XmlUserManager
user.manager.xml.file=conf/azkaban-users.xml
# Loader for projects
ddexecutor.global.properties=conf/global.properties
azkaban.project.dir=projects
database.type=mysql
mysql.port=3306
mysql.host=192.168.91.70
mysql.database=azkaban
mysql.user=hadoop
mysql.password=hadoop
mysql.numconnections=100
# Velocity dev mode
velocity.dev.mode=false
# Azkaban Jetty server properties.
jetty.use.ssl=false
jetty.maxThreads=25
jetty.port=11111
# Azkaban Executor settings
executor.port=12321
# mail settings
mail.sender=
mail.host=
job.failure.email=
job.success.email=
lockdown.create.projects=false
cache.directory=cache
# JMX stats
jetty.connector.stats=true
executor.connector.stats=true
权限配置文件azkaban-users.xml如下:
<azkaban-users>
<user username="admin" password="admin" roles="admin" groups="azkaban" />
<user username="metrics" password="metrics" roles="metrics"/>
<role name="admin" permissions="ADMIN" />
<role name="metrics" permissions="METRICS"/>
</azkaban-users>
增加文件log4j.properties:
vi /usr/local/azkaban/azkaban-web-server/conf/log4j.properties:
log4j.rootLogger=INFO,C
log4j.appender.C=org.apache.log4j.ConsoleAppender
log4j.appender.C.Target=System.err
log4j.appender.C.layout=org.apache.log4j.PatternLayout
log4j.appender.C.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
azkaban-exec-server配置
cd /usr/local/azkaban/azkaban-exec-server/conf
主要配置文件azkaban.properties:
#db set
database.type=mysql
mysql.port=3306
mysql.host=192.168.91.70
mysql.database=azkaban
mysql.user=hadoop
mysql.password=hadoop
mysql.numconnections=100
# Loader for projects
executor.global.properties=conf/global.properties
azkaban.project.dir=projects
# Azkaban Executor settings
executor.maxThreads=50
executor.port=12321
executor.flow.threads=30
# JMX stats
executor.connector.stats=true
# Azkaban plugin settings
azkaban.jobtype.plugin.dir=plugins/jobtypes
增加文件log4j.properties:
vi /usr/local/azkaban/azkaban-exec-server/conf/log4j.properties:
log4j.rootLogger=INFO,C
log4j.appender.C=org.apache.log4j.ConsoleAppender
log4j.appender.C.Target=System.err
log4j.appender.C.layout=org.apache.log4j.PatternLayout
log4j.appender.C.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
cd /usr/local/azkaban/azkaban-exec-server/plugins/jobtypes
主要配置文件commonprivate.properties:
# set execute-as-user
execute.as.user=false
azkaban启动与停止
azkaban因为启动和停止需要有关依赖故需要手动写脚本
su - 9euser
mkdir ~/script
vi ~/script/azkaban-web-start.sh 内容如下:
#!/bin/sh
source /etc/profile
cd /usr/local/azkaban/azkaban-web-server
./bin/azkaban-web-start.sh
vi ~/script/azkaban-web-shutdown.sh 内容如下:
#!/bin/sh
source /etc/profile
cd /usr/local/azkaban/azkaban-web-server
./bin/azkaban-web-shutdown.sh
vi ~/script/azkaban-exec-start.sh 内容如下:
#!/bin/sh
source /etc/profile
cd /usr/local/azkaban/azkaban-exec-server
./bin/azkaban-executor-start.sh
vi ~/script/azkaban-exec-shutdown.sh 内容如下:
#!/bin/sh
source /etc/profile
cd /usr/local/azkaban/azkaban-exec-server
./bin/azkaban-executor-shutdown.sh
chmod 777 ~/script/*
启动azkaban-web-server
sudo -u 9euser /home/9euser/script/azkaban-web-start.sh
启动azkaban-exec-server
sudo -u 9euser /home/9euser/script/azkaban-exec-start.sh
停止azkaban-web-server
sudo -u 9euser /home/9euser/script/azkaban-web-shutdown.sh
停止azkaban-exec-server
sudo -u 9euser /home/9euser/script/azkaban-exec-shutdown.sh
增加到开机启动
vi /etc/rc.local 增加如下内容:
sudo -u 9euser /home/9euser/script/azkaban-web-start.sh
sudo -u 9euser /home/9euser/script/azkaban-exec-start.sh