编写start.sh
#/bin/bash spark-submit --master yarn --name IotIoState5minJob --executor-memory 1G --executor-cores 1 --num-executors 3 --driver-memory 1G --conf "spark.ui.port=45053" --conf "spark.driver.extraJavaOptions=-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps" --class com.cserver.job.IotIoState5minJob /app/lib/spark-demo-1.0.jar
编写调度任务
vi /etc/crontab SHELL=/bin/bash PATH=/sbin:/bin:/usr/sbin:/usr/bin MAILTO=root # For details see man 4 crontabs # Example of job definition: # .---------------- minute (0 - 59) # | .------------- hour (0 - 23) # | | .---------- day of month (1 - 31) # | | | .------- month (1 - 12) OR jan,feb,mar,apr ... # | | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat # | | | | | # * * * * * user-name command to be executed */5 * * * * root su - hdfs -c /app/start.sh >> /app/spark.log