一、falcon磁盘IO告警计算方法
(1)线上告警示例
【falcon】环境: prod 时间: 2018-11-10 22:29 共1条
【#主机磁盘io过高(appid)】主机hostname磁盘dfa io过高98.76>98%
(2)cat /proc/diskstats
252 0 dfa 2689642164 0 513826977162 100403006 6803204529 0 1348198360088 2893131263 0 934780608 3002409596
每项的具体含义如下:
1 252 主设备号 code:Major
2 0 次设备号 code:Minor
3 dfa 设备名称 code:Name
4 2689642164 <1>读完成次数{读磁盘的次数,成功完成读的总次数}[2689642164] code:ReadIOs
5 0 <2>合并读完成次数 code:ReadMerges
6 513826977162 <3>读扇区的次数,成功读过的扇区总次数 5.ReadSectors
7 100403006 <4>读花费的毫秒数;这是所有读操作所花费的毫秒数{从__make_request()到end_that_request_last()的测量} code:ReadTicks
8 6803204529 <5>写完成次数;写完成的次数,成功写完成的总次数 code:WriteIOs
9 0 <6>合并写完成次数。为了效率可能会合并相邻的读和写,从而两次4K的读在它最终被处理到磁盘上之前可能会变成一次8K的读,才被计数(和排队),因此只有一次I/O操作,这个域使你知道这样的操作有多频繁 code:WriteMerges
10 1348198360088 <7>写扇区次数;写扇区的次数,成功写扇区总次数 code:WriteSectors
11 2893131263 <8>写操作花费的毫秒数;写花费的毫秒数,这是所有写操作所花费的毫秒数{是从__make_request()到end_that_request_last()的测量} code:WriteTicks
12 0 <9>正在处理的输入/输出请求数;I/O的当前进度,只有这个域应该是0.当请求被交给适当的request_queue_t时增加和请求完成时减小 code:InFlight
13 934780608 <10>输入/输出操作花费的毫秒数;花在I/O操作上的毫秒数,只要field9不为0这个域就会增长 code:IOTicks
14 3002409596 <11>输入/输出操作花费的加权毫秒数;花在I/O操作上的毫秒数,在每次I/O开始,I/O结束,I/O合并或读取自动上次更新这个域以来(第<9>列正在进行的io数量乘以花费在io上的毫秒数)时这个域都会增加.这可以给I/O完成时间和存储那些可以累积的提供一个便利的测量标准 code:TimeInQueue
(3)告警计算公式
io_ticks{crt.IOTicks - last.IOTicks(上10s的值)}/100 > 98 会进行报警
二、实现代码
(1)
const diskStatPath = "/proc/diskstats"
type DiskStatCollector struct {
lastDiskStats map[string]*linux.DiskStat
}
func (c *DiskStatCollector) Collect() ([]*model.Metric, error) {
if c.lastDiskStats == nil {
c.lastDiskStats = make(map[string]*linux.DiskStat)
}
disks, err := linux.ReadDiskStats(diskStatPath)
if err != nil {
return nil, fmt.Errorf("collect disk stat: %v", err)
}
var metrics []*model.Metric
for _, crt := range disks {
if len(crt.Name) == 3 && (strings.HasPrefix(crt.Name, "sd") || strings.HasPrefix(crt.Name, "vd") || strings.HasPrefix(crt.Name, "df")) {
if last, ok := c.lastDiskStats[crt.Name]; ok {
metrics = append(metrics, diskStatDiff(&crt, last)...)
}
tmp := crt
c.lastDiskStats[crt.Name] = &tmp
}
}
return metrics, nil
}
func diskStatDiff(crt *linux.DiskStat, last *linux.DiskStat) []*model.Metric {
var metrics []*model.Metric
if crt.IOTicks < last.IOTicks {
return metrics
}
metrics = append(metrics, &model.Metric{
Name: "disk.io",
Fields: []*model.Field{
{"read_ios", model.Gauge, crt.ReadIOs - last.ReadIOs},
{"read_merges", model.Gauge, crt.ReadMerges - last.ReadMerges},
{"read_sectors", model.Gauge, crt.ReadSectors - last.ReadSectors},
{"read_ticks", model.Gauge, crt.ReadTicks - last.ReadTicks},
{"write_ios", model.Gauge, crt.WriteIOs - last.WriteIOs},
{"write_merges", model.Gauge, crt.WriteMerges - last.WriteMerges},
{"write_sectors", model.Gauge, crt.WriteSectors - last.WriteSectors},
{"write_ticks", model.Gauge, crt.WriteTicks - last.WriteTicks},
{"in_flight", model.Gauge, crt.InFlight - last.InFlight},
{"io_ticks", model.Gauge, crt.IOTicks - last.IOTicks},
{"time_in_queue", model.Gauge, crt.TimeInQueue - last.TimeInQueue},
},
Tags: map[string]string{"name": crt.Name},
Timestamp: utils.Timestamp(),
})
return metrics
}
(2)
// ReadDiskStats reads and parses the file.
//
// Note:
// * Assumes a well formed file and will panic if it isn't.
func ReadDiskStats(path string) ([]DiskStat, error) {
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
devices := strings.Split(string(data), "
")
results := make([]DiskStat, len(devices)-1)
for i := range results {
fields := strings.Fields(devices[i])
Major, _ := strconv.ParseInt(fields[0], 10, strconv.IntSize)
results[i].Major = int(Major)
Minor, _ := strconv.ParseInt(fields[1], 10, strconv.IntSize)
results[i].Minor = int(Minor)
results[i].Name = fields[2]
results[i].ReadIOs, _ = strconv.ParseUint(fields[3], 10, 64)
results[i].ReadMerges, _ = strconv.ParseUint(fields[4], 10, 64)
results[i].ReadSectors, _ = strconv.ParseUint(fields[5], 10, 64)
results[i].ReadTicks, _ = strconv.ParseUint(fields[6], 10, 64)
results[i].WriteIOs, _ = strconv.ParseUint(fields[7], 10, 64)
results[i].WriteMerges, _ = strconv.ParseUint(fields[8], 10, 64)
results[i].WriteSectors, _ = strconv.ParseUint(fields[9], 10, 64)
results[i].WriteTicks, _ = strconv.ParseUint(fields[10], 10, 64)
results[i].InFlight, _ = strconv.ParseUint(fields[11], 10, 64)
results[i].IOTicks, _ = strconv.ParseUint(fields[12], 10, 64)
results[i].TimeInQueue, _ = strconv.ParseUint(fields[13], 10, 64)
}
return results, nil
}