搭建私有 helm 仓库 chartmuseum
chartmuseum 是一个开源的 Helm Chart Repository，支持多种后端存储，包括 GCS，S3 等。
#环境
1.k8s 1.16 集群
2.有nfs 可供挂载的存储
#创建 chartmuseum 私有仓库，准备配置文件
$ cat chartmuseum.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: chartmuseum
name: chartmuseum
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
app: chartmuseum
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
labels:
app: chartmuseum
spec:
containers:
- image: k8s-harbor01.gdfsxxds.rjyun/xy/chartmuseum:latest
imagePullPolicy: IfNotPresent
name: chartmuseum
ports:
- containerPort: 8080
protocol: TCP
env:
- name: DEBUG
value: "1"
- name: STORAGE
value: local
- name: STORAGE_LOCAL_ROOTDIR
value: /charts
resources:
limits:
cpu: 500m
memory: 256Mi
requests:
cpu: 100m
memory: 64Mi
volumeMounts:
- mountPath: /charts
name: charts-volume
volumes:
- name: charts-volume
nfs:
path: /data/upload/charts
server: 172.24.119.30
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: chartmuseum
namespace: kube-system
spec:
ports:
- port: 8080
protocol: TCP
targetPort: 8080
selector:
app: chartmuseum
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: chartmuseum
namespace: kube-system
spec:
rules:
- host: chart-xxx.com
http:
paths:
- path: /
backend:
serviceName: chartmuseum
servicePort: 8080
# 创建
kubectl apply -f chartmuseum.yaml
#检查
kubectl get pods -n kube-system
# 成功显示欢迎信息
浏览器访问 http://chart-xxx.com
# 添加 helm repo
helm repo add chartmuseum http://chart-xxx.com
helm repo list
创建应用并上传到 chartmuseum
#创建 chart
mkdir /root/helm && cd /root/helm
helm create myapp
#修改模板配置
cd /root/helm/myapp
$ tree
.
├── charts
├── Chart.yaml
├── templates
│ ├── deployment.yaml
│ ├── _helpers.tpl
│ ├── ingress.yaml
│ ├── NOTES.txt
│ ├── serviceaccount.yaml
│ ├── service.yaml
│ └── tests
│ └── test-connection.yaml
└── values.yaml
# values.yaml 配置
$ cat values.yaml
# Default values for deploy.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
log: ""
image:
repository: nginx
tag: stable
pullPolicy: IfNotPresent
imagePullSecrets: regcred
nameOverride: ""
fullnameOverride: ""
service:
type: ClusterIP
port: 80
annotations: {}
# prometheus.io/probe: "true"
# prometheus.io/port: "80"
ingress:
enabled: true
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/enable-cors: "true"
paths:
- /
hosts:
- chart-example.local
tls: false
## Configure extra options for liveness and readiness probes
## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes)
livenessProbe: {}
# tcpSocket:
# port: 80
# initialDelaySeconds: 3
# periodSeconds: 3
readinessProbe: {}
# httpGet:
# path: /
# port: 80
# initialDelaySeconds: 5
# periodSeconds: 5
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
nodeSelector: {}
tolerations: []
affinity: {}
## EXAMPLE:
## extraConfigmapMounts:
## - name: airflow-webserver-config
## mountPath: /etc/supervisord.conf
## configMap: laravel-config
## readOnly: true
## subPath: hotniuer2-config-production
##
extraConfigmapMounts: []
## EXAMPLE:
## services:
## - name: grpc
## port: 80
##
services: []
# Chart.yaml 配置
$ cat Chart.yaml
apiVersion: v1
appVersion: "1.0"
description: A Helm chart for Kubernetes
name: myapp
version: 0.9.0
#template 配置
deployment.yaml 配置：
*注意： deployment 名称是修改过的，
$ cat deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: "{{ .Release.Name }}-deployment"
labels:
app: {{ .Release.Name }}
helm.sh/chart: {{ include "deploy.chart" . }}
app.kubernetes.io/name: {{ include "deploy.fullname" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: "{{ .Release.Name }}"
template:
metadata:
labels:
app: "{{ .Release.Name }}"
log: "{{ .Values.log }}"
spec:
imagePullSecrets:
- name: {{ .Values.image.imagePullSecrets }}
containers:
- name: {{ .Release.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- range .Values.services }}
- name: {{ .name }}
containerPort: {{ .port }}
protocol: TCP
{{- end }}
{{- with .Values.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
env:
{{- range $key, $val := .Values.env }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
volumeMounts:
{{- range .Values.extraConfigmapMounts }}
- name: {{ .name }}
mountPath: {{ .mountPath }}
readOnly: {{ .readOnly }}
{{- if .subPath }}
subPath: {{ .subPath }}
{{- end }}
{{- end }}
terminationMessagePolicy: FallbackToLogsOnError
volumes:
{{- range .Values.extraConfigmapMounts }}
- name: {{ .name }}
configMap:
name: {{ .configMap }}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
#service 配置
*注意： service 名称是修改过的
$ cat service.yaml
apiVersion: v1
kind: Service
metadata:
name: "{{ .Release.Name }}-service"
labels:
app: {{ .Release.Name }}
helm.sh/chart: {{ include "deploy.chart" . }}
app.kubernetes.io/name: {{ include "deploy.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Values.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
{{- range .Values.services }}
- name: {{ .name }}
port: {{ .port }}
protocol: TCP
targetPort: {{ .port }}
{{- end }}
selector:
app: {{ .Release.Name }}
#ingress 配置
* 注意：{{- $fullName := .Release.Name -}} 是修改后的
$ cat ingress.yaml
{{- if .Values.ingress.enabled -}}
{{- $fullName := .Release.Name -}}
{{- $ingressPaths := .Values.ingress.paths -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: "{{ .Release.Name }}-ingress"
labels:
app: {{ .Release.Name }}
app.kubernetes.io/name: {{ include "deploy.name" . }}
helm.sh/chart: {{ include "deploy.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.ingress.tls }}
tls:
- hosts:
{{- range .Values.ingress.hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ $fullName }}-tls
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ . | quote }}
http:
paths:
{{- range $ingressPaths }}
- path: {{ . }}
backend:
serviceName: "{{ $fullName }}-service"
servicePort: http
{{- end }}
{{- end }}
{{- end }}
# 检查语法
cd /root/helm/myapp
helm lint myapp
# 打包
helm package myapp
# 上传
curl --data-binary "@myapp-0.9.0.tgz" http://chart-xxx.com/api/charts
通过 chartmuseum 安装软件
# 查找版本
helm search myapp
# 安装
helm install --name myapp chartmuseum/myapp
# 查看
helm list
helm status myapp
# 删除
helm del --purge myapp
更新 charts 并上传到 chartmuseum
cd /root/helm/myapp
vi values.yaml
# 调整 replicaCount 为 2
replicaCount: 2
vi Chart.yaml
# 把 version 改为 0.10.0
version: 0.10.0
# 测试，打包
cd /root/helm
helm lint myapp
helm package myapp
# 上传
curl --data-binary "@myapp-0.10.0.tgz" http://chart-xxx.com/api/charts
# 更新本地缓存然后可以查看已有的 charts：
helm repo update
helm search chartmuseum/
# 若之前没有删除，可以进行更新
helm upgrade myapp chartmuseum/myapp
# 回滚
helm rollback myapp 1
集合jenkins 使用
# upgrade -i 在k8s 集群上没有项目则创建，有项目则更新
helm upgrade -i mytestnginxhelm2 chartmuseum/myapp --set service.port=80,ingress={mytest-xxx.com},image.repository=k8s-harbor01.gdfsxxds.rjyun/xy/nginx,image.tag=stable --namespace=default