ci_guard

 1 #!/bin/bash
 2 # ./ci_guard.sh
 3 
 4 ERRORSTATUS=10
 5 TESTSTATUS=0
 6 USER="moonx"
 7 IPs="192.168.1.102 " # add host ip like "xx.xx.xx.xx xx.xx.xx.xx", delimit by space
 8 dic=([6]="192.168.1.102" [7]="xxx.xxx.xxx.xxx")
 9 webhookUri="https://hooks.slack.com/services/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"
10 lastrestart=`date +%s`
11 restart_gitlab_runner(){
12    for IP in ${IPs}
13    do
14        echo "restart gitlab-runner on ${IP}: ssh ${USER}@${IP}"
15        ssh ${USER}@${IP} "sudo service gitlab-runner restart; exit"
16        #ssh ${USER}@${IP} "date >> /usr/download/testcov/log; exit"
17        sleep 3
18    done
19 }
20 
21 sendmsgtoslack(){
22     runnerip=""
23     str=$*
24     arr=(${str// / })  
25     for i in ${arr[@]}  
26     do  
27         runnerip+=${dic[i]}" "
28     done
29     url="https://hooks.slack.com/services/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"
30     curdate="$(date "+%Y_%m_%d %H:%M:%S")"
31     #curl -v POST -H 'Content-type: application/json' -d '{
32     curl POST -H 'Content-type: application/json' -d '{
33       "response_type": "ephemeral",
34       "channel": "#demo",
35       "text": "CI Bug Report",
36       "username": "Bug News",
37       "icon_emoji": "",
38       "attachments": [{
39         "color": "#a6364f",
40         "title": "CI crashs.",
41         "title_link": "https://gitlab.moonx.cn/Developers/apollo/pipelines",
42         "text": "'"${curdate}"'",
43         "fields": [{
44           "title": "'"${runnerip}"'",
45         }]
46       }]
47     }' $url
48 }
49 
50 main(){
51     echo 'begin guard the ci.'
52     while true
53     do
54     # get pipelines info
55     pipelinesinfo=`curl --header "PRIVATE-TOKEN: UsDsmqSpivXMmNoCHcci" 
56                "https://gitlab.moonx.cn/api/v4/projects/2/pipelines" --insecure` # |python -m json.tool`
57     runnersinfo=`curl --header "PRIVATE-TOKEN: UsDsmqSpivXMmNoCHcci" 
58              "https://gitlab.moonx.cn/api/v4/projects/2/runners" --insecure`
59 
60     # resolve pipelines info
61     x=`python checkcistatus.py $pipelinesinfo $runnersinfo`
62     ci_status=$?
63     echo "python result:"$x
64 
65     if [ $ERRORSTATUS == $ci_status ]
66     then   
67        restart_gitlab_runner
68        tmptime=`date -d "-10 min" +%s`
69        if [ $tmptime -lt $lastrestart ]
70        then
71            sendmsgtoslack $x
72        fi
73        lastrestart=`date +%s`
74     fi
75     #sendmsgtoslack $x
76     sleep 300
77     done
78     echo 'exit guard the ci.'
79 }
80 
81 main

ci_guard.sh

 1 # coding=utf-8
 2 # python checkcistatus.py $pipelinesinfo $runnersinfo
 3 
 4 import sys
 5 import re
 6 
 7 ERRORSTATUS=10
 8 
 9 pipeidstr='"id":'
10 pendingpipe='"status":"pending"'
11 runningpipe='"status":"running"'
12 
13 template_str_runner='"name":"gitlab-runner"'
14 template_str_active_runner='"active":true'
15 
16 if __name__ == "__main__":
17     if len(sys.argv) < 3:
18         print "usage: python checkcistatus.py $pipelinesinfo $runnersinfo"
19         sys.exit(1)
20 
21 #for i in range(0,len(sys.argv)):
22 #    print(i, sys.argv[i])
23 pipe_num=len(re.findall(pipeidstr, sys.argv[1]))
24 runningpipe_num=len(re.findall(runningpipe, sys.argv[1]))
25 pendingpipe_num=len(re.findall(pendingpipe, sys.argv[1]))
26 #print "pipe_num:%d,running:%d,pending:%d" % (pipe_num,runningpipe_num,pendingpipe_num)
27 
28 runner_num=len(re.findall(template_str_runner, sys.argv[2]))
29 active_runner_num=len(re.findall(template_str_active_runner, sys.argv[2]))
30 reg = re.compile('{"id":([^,]*),"description":"([^,]*)","active":([^,]*),"is_shared":([^,]*),"name":"([^,]*)","online":([^,]*),"status":"([^,]*)"}')
31 runners = re.findall(reg,sys.argv[2])
32 for runner in runners:
33     #print runner
34     if runner[6] == 'online':
35        print runner[0]
36 #print "runner:%d,active_runner:%d
" % (runner_num,active_runner_num)
37 
38 if 0==runningpipe_num and 0!=pendingpipe_num:
39     #print "[error]no running pipeline, pending pipeline exists."
40     #print "[error]need to restart gitlab-runner."
41     sys.exit(ERRORSTATUS)
42 
43 if 0!=pendingpipe_num and runningpipeline_num < active_runner_num:
44     #print "[error]running pipeline less than active runner, pending pipeline exists."
45     #print "[error]need to restart gitlab-runner."
46     sys.exit(ERRORSTATUS) 
47 
48 #print "[info]not need to restart gitlab-runner."
49 sys.exit(0)

checkcistatus.py

 1 #coding:utf-8
 2 #server.login(my_sender,"ydydmvthorlqbgbg") my_sender='490195356@qq.com'
 3 #coding=utf-8
 4 import smtplib
 5 from email.mime.text import MIMEText
 6 msg_from='490195356@qq.com'
 7 passwd='ydydmvthorlqbgbg'
 8 msg_to="490195356@qq.com,peng.yang@moonx.ai"
 9 subject="no reply<python邮件测试>"    
10 content='''
11 #coding:utf-8
12 #server.login(my_sender,"ydydmvthorlqbgbg") my_sender='490195356@qq.com'
13 #coding=utf-8
14 import smtplib
15 from email.mime.text import MIMEText
16 msg_from='490195356@qq.com'
17 passwd='ydydmvthorlqbgbg'
18 msg_to="490195356@qq.com,peng.yang@moonx.ai"
19 subject="no reply<python邮件测试>"    
20 content="pp"
21 msg = MIMEText(content)
22 msg['Subject'] = subject
23 msg['From'] = msg_from
24 msg['To'] = msg_to
25 try:
26     s = smtplib.SMTP_SSL("smtp.qq.com",465)
27     s.login(msg_from, passwd)
28     s.sendmail(msg_from, msg_to.split(','), msg.as_string())
29     print "发送成功"
30 except s.SMTPException,e:
31     print "发送失败"
32 finally:
33     s.quit()  
34 '''
35 msg = MIMEText(content)
36 msg['Subject'] = subject
37 msg['From'] = msg_from
38 msg['To'] = msg_to
39 try:
40     s = smtplib.SMTP_SSL("smtp.qq.com",465)
41     s.login(msg_from, passwd)
42     s.sendmail(msg_from, msg_to.split(','), msg.as_string())
43     print "发送成功"
44 except s.SMTPException,e:
45     print "发送失败"
46 finally:
47     s.quit()

sendmailtoa.py

gitlab + gitlab-ci + slack 持续集成框架

Using curl POST with variables defined in bash script functions

https://stackoverflow.com/questions/17029902/using-curl-post-with-variables-defined-in-bash-script-functions