#!/bin/bash
size="120Y*10W"
date1=`date +%F_%H-%M-%S`
config="spark-submit
--jars /data0/work_space/service/spark-2.0.0-bin-hadoop2.4/jars/hadoop-lzo-0.4.15.jar
--master yarn
--deploy-mode client
--num-executors 100
--driver-memory 7g
--executor-cores 4
--executor-memory 7g
--class com.weibo.datasys.pipeline.Runner
weispark-ml-0.5.0-SNAPSHOT.jar pipeline.xml [5]
"
echo $config
echo ===========================================
spark-submit
--jars /data0/work_space/service/spark-2.0.0-bin-hadoop2.4/jars/hadoop-lzo-0.4.15.jar
--master yarn
--deploy-mode client
--num-executors 100
--driver-memory 7g
--executor-cores 4
--executor-memory 7g
--conf spark.driver.maxResultSize=3g
--conf spark.ui.retainedJobs=2
--conf spark.ui.retainedStages=2
--conf spark.worker.ui.retainedExecutors=5
--conf spark.worker.ui.retainedDrivers=5
--conf spark.eventLog.enabled=false
--class com.weibo.datasys.pipeline.Runner
weispark-ml-0.5.0-SNAPSHOT.jar pipeline.xml [5]
date2=`date +%F_%H-%M-%S`
echo ===========================================
echo date start : $date1
echo data size : $size
echo config : $config
echo date ended : $date2