CMDB - 发布系统
完全自己开发一套发布系统
表设计
- 环境，
- 主机 ， -> salt-id
- 代码地址、就是包 -> 地址
- 应用 -> app
-记录日志 -> 时间，事件
SaltStack
SaltStack 采用 C/S模式
master和minion之间的通信用到了zeromq消息队列 ，每个minion 有一个salt_id 是绝对唯一的
Master与Minion之间通过ZeroMq进行消息传递，使用了ZeroMq的发布-订阅模式，连接方式包括tcp，ipc
废话少说，直接上图：
发布系统流程图.jpg
表设计
models.py
from django.db import models
# Create your models here.
class Use_Env(models.Model):
name = models.CharField(max_length=32, blank=True, null=True, verbose_name='环境名')
def __str__(self):
return self.name
class Meta:
verbose_name_plural = '环境表'
class Host(models.Model):
hostname = models.CharField(max_length=32, blank=True, null=True, verbose_name="salt_id")
ip = models.CharField(max_length=64, blank=True, null=True, verbose_name='IP')
def __str__(self):
return self.hostname
class Meta:
verbose_name_plural = "主机表"
class Record_Log(models.Model):
timestamp = models.CharField(max_length=64, blank=True, null=True, verbose_name='时间')
project = models.ForeignKey(to='App', blank=True, null=True, verbose_name='项目', related_name='proj')
package = models.ManyToManyField(to='Package', blank=True, null=True, verbose_name='包', related_name='pack')
env = models.ForeignKey(to='Use_Env', blank=True, null=True, verbose_name='环境', related_name='env')
def __str__(self):
return self.timestamp
class Meta:
verbose_name_plural = '记录日志'
class App(models.Model):
name = models.CharField(max_length=32, blank=True, null=True, verbose_name='应用名')
path = models.CharField(max_length=64, blank=True, null=True, verbose_name='应用路径')
environment = models.ForeignKey(to='Use_Env', blank=True, null=True, verbose_name='环境')
hosts = models.ForeignKey(to='Host', blank=True, null=True, verbose_name='对应主机', related_name='apphost')
# _script = models.CharField(max_length=32, blank=True, null=True, verbose_name='部署脚本')
package = models.ForeignKey(to='Package', blank=True, null=True, verbose_name='代码', related_name='apppack')
_app = models.ForeignKey(to='App', blank=True, null=True, verbose_name='上级应用')
def __str__(self):
return self.name
class Meta:
verbose_name_plural = '项目表'
class Package(models.Model):
name = models.CharField(max_length=64, blank=True, null=True, verbose_name='包名/版本号')
pack_path = models.CharField(max_length=64, blank=True, null=True, verbose_name='代码路径/地址')
project = models.ForeignKey(to='App', blank=True, null=True, verbose_name='所属项目', related_name='packapp')
def __str__(self):
return self.name
class Meta:
verbose_name_plural = '代码'
迁移，创建数据库python manage.py makemigrationspython manage.py migrate
伪代码
views.py
def pubilsh(request):
if request.method == 'GET':
env = models.Use_Env.object.all()
return render(request,'fabu.html', locals())
else:
env = request.POST.get('env')
app = request.POST.get('app')
obj_list = models.App.objects.filter(name=app,environment__name=env) # 跨表查询
# 拿到对应的主机组 代码 -> 地址
# 循环 主机组 推送代码
app_name = ORM 查表
host_list = [{'id':'salt-id','path':'/data/app/www/abc'},] # 这里是通过数据库取到的
package = 'svn://xxxx' # svn 地址
template
fabu.html
<form method='post'>
<label>应用：</label>
<input type="text" name="app"> # name -> key ,框是 -> values
<select class="form-control" id="nubers" name="env"> # name="env" -> key
{% for i in env %}
<option value="{{ i.name }}"> {{ i.name }}</option> # value="{{ i.name }}" -> values
{% endfor %}
</select>
<input type="submit" value="提交">
</form>
自动化管理平台 -> 必须是 和salt-master 安装在同一台机上 ，使用salt原生的API
第一步 在 自动化管理平台 里面下载代码 （可打包） 通过 subporcsess 执行命令
# from subprocess import Popen, PIPE
import os
path = os.getcwd() + r'/project_path/'
subprocsess -> 执行命令
# cd path
# mkdir app_name
# svn co $package
# tar 打包
create.sh
#!/bin/bash
cd path
mkdir $app_name && cd $app_name
svn co $package
tar 打包
subprocess.call(['cd', '-l'])
from subprocess import Popen, PIPE
p = subprocess.Popen('sh create.sh', stdout=PIPE, shell=True)
第二步 推送 salt stack -> state.sls # 状态管理
写 state.sls 规则的yml文件
通过 Python 代码 salt-api 调用 state 触发推送
第三部 执行远程端代码 -> cmd.run cd 路径 python xxx
django celery 被封装成了 djcelery
就要学会如何使用
celery.py
form __future__ import absolute_import, unicode_literals
import os
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTIONG_MODULE', '项目名称.settiongs')
app = Celery('项目名称')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
@app.task(bind=True)
def debug_task(self):
print ('Request: {0!x}'.format(self.request))
settings.py
# 文件最后添加
import djcelery
from celery.schedules improt crontab
from datetime import timedelta
djcelery.setup_loader()
CELERY_TIMEZONE = TIME_ZONE
BROKER_URL='redis://:' # redis 地址 发送端口
CELERY_RESULT_BACKEND = 'redis://:' # redis 接收端口
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Africa/Nairobi'
CELERY_IMPORTS = ['应用名目录下的.task',] # 应用名目录下的.task ，主要看有没有task.py文件
CELERY_MAX_TASKS_PRR_CHILD = 3
CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
task.py
form __future__ import absolute_import, unicode_literals
import time
import requests
from celery import shared_task
from django.views.decorators.csrf import csrf_exempt, csrf_protect
from django.shortcuts import render, HttpResponse,redirect
@shared_task:
def add(x, y):
return x+y # 定义自己的推送代码
@shared_task:
def add(x, y):
return x*y
@shared_task:
def xsun(numbers):
print (sum(numbers))
return sum(numbers)
urls.py
urlpatterns = [
url(r'^celery/', views.celery_status), # 必须要写的路由
]
views.py
def celery_status(request):
import datetime
import json
if request.method == 'GET':
if request.GET.get('x') and request.GET.get('y'):
if request.GET.get('after'):
ctime = datetime.datetime.now()
utc_ctime = datetime.datetime.utcfromtimestamp(ctime.timestamp())
s1 = datetime.timedelta(seconds=int(request.GET.get('after'))*60)
ctime_x = utc_ctime + s1
year = request.GET.get('year')
mouth = request.GET.get('month')
day = reuqest.GET.get('day')
hour = request.GET.get('hour')
minute = request.GET.get('minute')
if year and mouth and day and hour and minute:
ctime = datetime.datetime(year=int(year), month=int(mouth)),
day=int(day), hour=int(hour), minute=int(minute))
# 把当前的本地时间转换成 UTC 时间
ctime_x = datetime.datetime.utcfromtimestamp(ctime.timestamp)
if ctime_x:
# 最核心的代码
ret = add.apply_async(args=[int(request.GET.get('x')), int(request.GET.get('y'))], eta=ctime_x)
num = ret.id
if request.GET.get('cancel'):
async = AsyncResult(id=request.GET.get('cancel'), app=app)
async.revoke(terminate=True)
cancel_tag = '取消成功'
if request.GET.get('stop'):
async = AsyncResult(id=request.GET.get('stop'), app=app)
async.revoke()
stop_tag='中止成功'
return render(request, 'celery.html', locals())
else:
ret = request.POST.get('id','')
data = ""
if ret:
async = AsyncResult(id=ret,app=app)
if async.successful():
data = "执行成功，数据是: " + str( async.get() )
async.forget()
elif async.failed():
data='执行失败'
elif async.status == 'PBNDING':
data = "等待被执行"
elif async.status == 'RBTPY' :
data = '任务异常正常重试'
elif async.status == 'STARTBD':
data = "任务正在执行"
else:
data = "未知"
retrun render(request, 'celery.html', locals())
celery 需要在命令行里单独启动 terminal
celery worker -A 发布 -l debug
templates
celery.html
<form method="post">
{% csrf_token %}
id: <input type="text" name='id'>
结果: <input type="text" value="{{ data }}">
<input type="submit" value="提交">
</form>
<br> # 空行
<hr> # 分割线
<form method="get">
x:<input type="text" name="x">
+
y:<input type="text" name="y">
<br>
年: <input type="text" name="year">
月: <input type="text" name="month">
日: <input type="text" name="day">
时: <input type="text" name="hour">
分: <input type="text" name="minute">
<br>
几分钟后： <inpur type="text" name="after">
<br>
取消这个任务: <input type="text" name="cancel">
结果: <input type="text" value="{{ cancel_tag }}">
<br>
中止这个任务: <input type="text" name="stop">
结果: <input type="text" value="{{ stop_tag }}">
<br>
<hr>
结果: <input type="text" value="{{ stop_tag }}"
<br>
<hr>
结果： <input type="text" value="{{ num }}">
<input type="submit" value="提交">
</form>
发布代码
第一步 在 自动化管理平台 里面下载代码 （可打包） 通过 subporcsess 执行命令
# from subprocess import Popen, PIPE
import os
path = os.getcwd() + r'/project_path/'
subprocsess -> 执行命令
# cd path
# mkdir $app_name && cd $app_name
# svn co $package
# tar 打包
create.sh
#!/bin/bash
cd path
mkdir $app_name && cd $app_name
svn co $package
tar 打包
from subprocess import Popen, PIPEpath = os.getcwd() + r'/project_path/'
拼接
cd path && mkdir $app_name && cd $app_name && svn co $package
cmd = 'cd {0} && mkdir {1} && cd {1} && svn co {2}'.format(path, app_name, package)
第二步 推送 salt stack -> state.sls # 状态管理
写 state.sls 规则的yml文件
通过 Python 代码 salt-api 调用 state 触发推送
第三部 执行远程端代码 -> cmd.run cd 路径 python xxx
class MainSalt(object): # salt 代码
def __init__(self, tgt='*')
self.local = sc.LocalClient()
self.tgt = tgt
def get_cache_returns(self, func):
while not self.local.get_cache_returns(func):
time.sleep(1)
return self.local.get_cache_returns(func)
def cmd_run(self, run_func):
if not isinstance(run_func, list):
raise TypeError(AttributeError)
cmd_id = self.local.cmd_async(self.tgt, 'cmd.run', run_func)
ret_cmd = self.get_cache_returns(cmd_id)
return ret_cmd
def state(self, salt_fun, tag=''):
if tag:
disk_id = self.local.cmd_async(self.tgt, 'state.sls', [salt_fun, tag])
# 实际上 把信息塞入ZeroMq 返回一个ID
else:
disk_id = self.local.cmd_async(self.tgt, 'state.sls', [salt_fun,])
ret_disk_data = self.get_cache_returns(disk_id)
return ret_disk_data
def push_package(self, pillar_dic):
tag = 'pillar={0}'.format(json.dumps(pillar_dic))
salt_fun = 'test' # test.sls 就是状态管理里面的这个文件 加载
return self.state(salt_fun, tag)
def sub_run(cmd):
retrue subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
def pubilsh(request):
if request.method == 'GET':
env = models.Use_Env.object.all()
return render(request,'fabu.html', locals())
else:
env = request.POST.get('env')
app = request.POST.get('app')
obj_list = models.App.objects.filter(name=app,environment__name=env) # 跨表查询
# 拿到对应的主机组 代码 -> 地址
# 循环 主机组 推送代码
app_name = ORM 查表
host_list = [{'id':'salt-id','path':'/data/app/www/abc'},] # 这里是通过数据库取到的
package = 'svn://xxxx' # svn 地址
for i in host_list:
cmd = 'cd {0} && mkdir {1} && cd {1} && svn co {2}'.format(path, app_name, package)
ret = sub_run(cmd)
m_salt = MainSalt(host.get('id'))
pillar_dic = {
'path':i.get('path')+ '/' + app_name,
'app' : app_name
}
ret = m_salt.push_package(pillar_dic)
#第三部
m_salt.cmd_run('cd {0} && python manage.py runserver 8080'.format(itme.get('path') + '/' + app_name))
satlstack 管理
top.sls
base:
'*':
- jar_package
test.sls
test_ci:
file.recurse:
- name: {{ pillar['path'] }}
- source: salt://project_path/{{ pillar['app'] }} # project_path需要做个软连接
- user: root
- dir_mode: 755
- file_mode: 644
- template: jinja
- makedirs: True
- include_enpty: True