Go语言学习之12 etcd、contex、kafka消费实例、logagent

本节内容：
    1. etcd介绍与使用
    2. ElastcSearch介绍与使用

1. etcd介绍与使用
    概念：高可用的分布式key-value存储，可以使用配置共享和服务发现
    类似项目：zookeeper和consul
    开发语言：Go
    接口：提供restful的http接口，使用简单
    实现算法：基于raft算法的强一致性、高可用的服务存储目录

2. etcd的应用场景
    a. 服务发现和服务注册
    b. 配置中心
    c. 分布式存储
    d. master选举

3. etcd搭建
    a. 下载etcd release版本：https://github.com/etcd-io/etcd/releases 版本
    b. 解压后，进入到etcd的根目录，直接执行./etcd 可以启动etcd
    c. 使用etcdctl工具更改配置

4. context使用介绍
    a. 如何控制goroutine
    b. 如何保存上下文数据

 （1）使用context处理超时
    ctx, cancel = context.With.Timeout(context.Background(), 2*time.Second)

    示例是通过设置ctx超时时间为2s，如果2s类无法接收到baidu的请求返回，则超时异常。

package main

import (
    "context"
    "fmt"
    "io/ioutil"
    "net/http"
    "time"
)
type Result struct {
    r   *http.Response
    err error
}

func process() {
    ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
    defer cancel()
    tr := &http.Transport{}
    client := &http.Client{Transport: tr}
    c := make(chan Result, 1)
    req, err := http.NewRequest("GET", "http://www.baidu.com", nil)
    if err != nil {
        fmt.Println("http request failed, err:", err)
        return
    }
    go func() {
        resp, err := client.Do(req)
        pack := Result{r: resp, err: err}
        c <- pack
    }()
    select {
    case <-ctx.Done():
        tr.CancelRequest(req)
        res := <-c
        fmt.Println("Timeout! err:", res.err)
    case res := <-c:
        defer res.r.Body.Close()
        out, _ := ioutil.ReadAll(res.r.Body)
        fmt.Printf("Server Response: %s", out)
    }
    return
}
func main() {
    process()
}

（2） 使用context保存上下文
    利用context来保存上下文值：

package main

import (
    "context"
    "fmt"
)

func process(ctx context.Context) {
    ret,ok := ctx.Value("trace_id").(int)
    if !ok {
        ret = 21342423
    }

    fmt.Printf("ret:%d
", ret)

    s , _ := ctx.Value("session").(string)
    fmt.Printf("session:%s
", s)
}

func main() {
    ctx := context.WithValue(context.Background(), "trace_id", 13483434)
    ctx = context.WithValue(ctx, "session", "sdlkfjkaslfsalfsafjalskfj")
    process(ctx)
}

    同时还有context ctx_cancel 和 ctx_deadline

package main

import (
    "context"
    "fmt"
    "time"
)

func gen(ctx context.Context) <-chan int {
    dst := make(chan int)
    n := 1
    go func() {
        for {
            select {
            case <-ctx.Done():  //当43行的test函数执行结束之后，执行defer cancel()，则会触发该行
                fmt.Println("i exited")
                return // returning not to leak the goroutine
            case dst <- n:
                n++
            }
        }
    }()
    return dst
}

func test() {
    // gen generates integers in a separate goroutine and
    // sends them to the returned channel.
    // The callers of gen need to cancel the context once
    // they are done consuming generated integers not to leak
    // the internal goroutine started by gen.
    ctx, cancel := context.WithCancel(context.Background())
    defer cancel() // cancel when we are finished consuming integers
    intChan := gen(ctx)
    for n := range intChan {
        fmt.Println(n)
        if n == 5 {
            break
        }
    }
}
func main() {
    test()
    time.Sleep(time.Hour)
}

package main

import (
    "context"
    "fmt"
    "time"
)

func main() {
    d := time.Now().Add(50 * time.Millisecond)
    ctx, cancel := context.WithDeadline(context.Background(), d)

    // Even though ctx will be expired, it is good practice to call its
    // cancelation function in any case. Failure to do so may keep the
    // context and its parent alive longer than necessary.
    defer cancel()

    select {
    case <-time.After(1 * time.Second):
        fmt.Println("overslept")
    case <-ctx.Done():
        fmt.Println(ctx.Err())  //context deadline exceeded
    }

}

5. etcd介绍与使用
    etcd使用示例 （由于虚拟机出现问题，后面的程序全在Windows上面操作）：

（1）客户端连接 etcd server端

package main

import (
    "fmt"
    //etcd_client "github.com/coreos/etcd/clientv3"
    etcd_client "go.etcd.io/etcd/clientv3"
    "time"
)

func main() {

    cli, err := etcd_client.New(etcd_client.Config{
        Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
        DialTimeout: 5 * time.Second,
    })
    if err != nil {
        fmt.Println("connect failed, err:", err)
        return
    }

    fmt.Println("connect succ")
    defer cli.Close()
}

（2）put 和 get

package main

import (
    "context"
    "fmt"
    "go.etcd.io/etcd/clientv3"
    "time"
)

func main() {

    cli, err := clientv3.New(clientv3.Config{
        Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
        DialTimeout: 5 * time.Second,
    })
    if err != nil {
        fmt.Println("connect failed, err:", err)
        return
    }

    fmt.Println("connect succ")
    defer cli.Close()
    //设置1秒超时，访问etcd有超时控制
    ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    //操作etcd
    _, err = cli.Put(ctx, "/logagent/conf/", "192.168.0.1")
    //操作完毕，取消etcd
    cancel()
    if err != nil {
        fmt.Println("put failed, err:", err)
        return
    }
    //取值，设置超时为1秒
    ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
    resp, err := cli.Get(ctx, "/logagent/conf/")
    cancel()
    if err != nil {
        fmt.Println("get failed, err:", err)
        return
    }
    for _, ev := range resp.Kvs {
        fmt.Printf("%s : %s
", ev.Key, ev.Value)
    }
}

（3）watch（观测key值发生变化）

package main

import (
    "context"
    "fmt"
    "time"

    etcd_client "go.etcd.io/etcd/clientv3"
)

func main() {

    cli, err := etcd_client.New(etcd_client.Config{
        Endpoints:   []string{"localhost:2379", "localhost:22379", "localhost:32379"},
        DialTimeout: 5 * time.Second,
    })
    if err != nil {
        fmt.Println("connect failed, err:", err)
        return
    }
    defer cli.Close()

    fmt.Println("connect succ")
    
    ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    _, err = cli.Put(ctx, "/logagent/conf/", "99999")
    if err != nil {
        fmt.Println("put failed, err:", err)
        return
    }
    cancel()
    
    fmt.Println("put succ")

    for {
        rch := cli.Watch(context.Background(), "/logagent/conf/")
        for wresp := range rch {
            for _, ev := range wresp.Events {
                fmt.Printf("%s %q : %q
", ev.Type, ev.Kv.Key, ev.Kv.Value)
            }
        }
    }
}

    运行上面的watch程序监控key（/logagent/conf/）操作的变化，然后再运行（2）的程序，结果如下：

    kafka消费示例代码：

package main

import (
    "fmt"
    "strings"
    "sync"

    "github.com/Shopify/sarama"
)

var (
    wg sync.WaitGroup
)

func main() {

    consumer, err := sarama.NewConsumer(strings.Split("192.168:30.136:9092", ","), nil)
    if err != nil {
        fmt.Println("Failed to start consumer: %s", err)
        return
    }
    partitionList, err := consumer.Partitions("nginx_log")
    if err != nil {
        fmt.Println("Failed to get the list of partitions: ", err)
        return
    }

    fmt.Println(partitionList)

    for partition := range partitionList {
        pc, err := consumer.ConsumePartition("nginx_log", int32(partition), sarama.OffsetNewest)
        if err != nil {
            fmt.Printf("Failed to start consumer for partition %d: %s
", partition, err)
            return
        }
        defer pc.AsyncClose()
        
        go func(pc sarama.PartitionConsumer) {
            wg.Add(1)
            for msg := range pc.Messages() {
                fmt.Printf("Partition:%d, Offset:%d, Key:%s, Value:%s", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
                fmt.Println()
            }
            wg.Done()
        }(pc)
    }
    //time.Sleep(time.Hour)
    wg.Wait()
    consumer.Close()
}

6. sync.WaitGroup介绍
1）等待一组groutine结束
2）使用Add方法设置等待的数量加1
3）使用Delete方法设置等待的数量减1
4）当等待的数量等于0，Wait函数返回

sync.WaitGroup实例：

package main

import (
    "fmt"
    "sync"
    "time"
)

func main() {
    wg := sync.WaitGroup{}
    for i := 0; i < 10; i++ {
        wg.Add(1)
        go calc(&wg, i)
    }

    wg.Wait() //阻塞，等待所有groutine结束
    fmt.Println("all goroutine finish")
}

func calc(w *sync.WaitGroup, i int) {
    //注意： wg.Add(1) 放到这会有问题，也就是main函数结束比wg.Add(1)要快
    fmt.Println("calc:", i)
    time.Sleep(time.Second)
    w.Done()
}

7. ElastcSearch安装及go操作es

（1）安装 es
   1）下载ES，下载地址：https://www.elastic.co/products/elasticsearch，我下载的是 elasticsearch-6.7.1.zip。
   2）修改在解压后根目录下的 /config/elasticsearch.yml 配置：

    放开注释并将 youIP换成你自己机器的 ip

cluster.name: my-application
node.name: node-1
network.host: youIP
http.port: 9200

   3）修改 /config/jvm.options 文件，当然如果机器性能好也可以不用修改：

-Xms512m
-Xmx512m

   4）进入根目录，启动es，.inelasticsearch.bat

（2）go 操作 es 示例

    安装第三方插件：

go get gopkg.in/olivere/elastic.v2

    示例：注意将程序里面的 url = "http://yourIP:9200/"，yourIP替换为你安装es机器的 ip：

package main

import (
    "fmt"

    elastic "gopkg.in/olivere/elastic.v2"
)

type Tweet struct {
    User    string
    Message string
}

var (
    url = "http://yourIP:9200/"
)

func main() {
    client, err := elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(url))
    if err != nil {
        fmt.Println("connect es error", err)
        return
    }

    fmt.Println("conn es succ")

    tweet := Tweet{User: "olivere", Message: "Take Five"}
    _, err = client.Index().
        Index("twitter").
        Type("tweet").
        Id("1").
        BodyJson(tweet).
        Do()
    if err != nil {
        // Handle error
        panic(err)
        return
    }

    fmt.Println("insert succ")
}

    链式存储：

package main

import "fmt"

type Stu struct {
    Name string
    Age  int
}

func (p *Stu) SetName(name string) *Stu {
    p.Name = name
    return p
}

func (p *Stu) SetAge(age int) *Stu {
    p.Age = age
    return p
}

func (p *Stu) Print() {
    fmt.Printf("age:%d name:%s
", p.Age, p.Name)
}

func main() {
    stu := &Stu{}
    stu.SetAge(12).SetName("stu01").Print()
    //stu.SetName("stu01")
    //stu.Print()
}