4

oozie 调用spark-转载

 3 years ago
source link: http://www.hechunbo.com/index.php/archives/379.html
Go to the source link to view the article. You can view the picture content, updated content and better typesetting reading experience. If the link is broken, please click the button below to view the snapshot at that time.

oozie 调用spark

1.workflow.xml

<workflow-app  name="Spark-example2" xmlns="uri:oozie:workflow:0.5">  
    <start to="SparkOozieAction1"/>  
    <action name="SparkOozieAction1">
        <spark xmlns="uri:oozie:spark-action:0.1">  
            <job-tracker>${jobTracker}</job-tracker>  
            <name-node>${nameNode}</name-node>
            <configuration>  
                <property>  
                    <name>mapred.job.queue.name</name>  
                    <value>${queueName}</value>  
                </property>                  
            </configuration>            
            <master>yarn-cluster</master>
            <mode>cluster</mode>
            <name>Spark Example1</name>
            <class>com.ocn.itv.rinse.ErrorCollectRinse</class>            
            <jar>ocn-itv-spark-3.0.3-rc1.jar</jar> 
            <spark-opts>${sparkopts}</spark-opts> 
            <arg>${input1}</arg>            
        </spark > 
        <ok to="end"/>
        <error to="kill"/>    
    </action>  
    <kill name="kill">  
        <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>  
    </kill>  
    <end name="end"/>  
</workflow-app>

job.properties

nameNode=hdfs://hgdp-001:8020    --HDFS文件系统的端口
jobTracker=hgdp-001:8032   resourceManager的端口
queueName=default    --oozie队列
input1=2017-05-09    --要输入的参数
hdfspath=user/root  --自定义路径
examplesRoot=ocn-itv-oozie    ---全局目录
oozie.use.system.libpath=True  
sparkopts=--executor-memory 1G    --是否加载用户lib库(oozie的system share lib)
oozie.libpath=${nameNode}/${hdfspath}/${examplesRoot}/lib/    用户自定义lib库(存放所需jar包)
oozie.wf.application.path=${nameNode}/${hdfspath}/${examplesRoot}/wf/wf2/   --workflow.xml在hdfs中的地址

参考:http://cache.baiducontent.com/c?m=9b1Y1DgFp7vpE0xeHvKnM8yLCAbb2FPelEVgEp9axirY1rWHe4eqTK0pfOtAEBvUsL4_nx4cGTl_H5-fFD5GF8F4p8L1qRxfwjWRSOFNU8hJdDK965gN8SZooBT9JfBzhMc51CUCN7lUhfJFfYSFNFI7yebJzjdMukXQ2dSfKiC&p=c336c54ad5c34bed0cf5c52d0214cc&newp=b4759a46d6c91fdd09aac7710f0c95231610db2151d7d3156b82c825d7331b001c3bbfb423291704d6ce7e6c0ba94b5de9f53771330923a3dda5c91d9fb4c57479d6556537&s=c81e728d9d4c2f63&user=baidu&fm=sc&query=OOZIE+spark+jar%B0%FC&qid=ed65eb0900084c0e&p1=3


About Joyk


Aggregate valuable and interesting links.
Joyk means Joy of geeK