[root@bs-k8s-ceph ~]# ceph -s cluster: id: 11880418-1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: 3 daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-hk-hk02(active), standbys: bs-k8s-ceph, bs-hk-hk01 osd: 6 osds: 6 up, 6 in data: pools: 3 pools, 320 pgs objects: 416 objects, 978 MiB usage: 8.7 GiB used, 105 GiB / 114 GiB avail pgs: 320 active+clean [root@bs-k8s-master01 ~]# kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE default rbd-provisioner-75b85f85bd-8ftdm 0/1 Terminating 12 5d default rbd-provisioner-75b85f85bd-bk5kz 1/1 Running 4 24h harbor rbd-provisioner-75b85f85bd-dhnr4 1/1 Terminating 7 3d13h harbor rbd-provisioner-75b85f85bd-qzzbx 1/1 Running 3 24h kube-system calico-kube-controllers-5b644bc49c-kv4wb 1/1 Running 67 10d kube-system calico-node-4jxbp 1/1 Running 5 10d kube-system calico-node-7t9cj 1/1 Running 12 10d kube-system calico-node-cchgl 1/1 Running 15 10d kube-system calico-node-czj76 1/1 Running 7 10d kube-system calico-node-lxb2s 0/1 Running 15 10d kube-system calico-node-nmg9t 1/1 Running 8 10d kube-system coredns-7f9c544f75-bwx9p 1/1 Running 67 10d kube-system coredns-7f9c544f75-q58mr 1/1 Running 66 10d kube-system dashboard-metrics-scraper-6b66849c9-qtwzx 1/1 Running 83 10d kube-system etcd-bs-k8s-master01 1/1 Running 30 10d kube-system etcd-bs-k8s-master02 1/1 Running 12 10d kube-system etcd-bs-k8s-master03 1/1 Running 37 10d kube-system kube-apiserver-bs-k8s-master01 1/1 Running 49 10d kube-system kube-apiserver-bs-k8s-master02 1/1 Running 22 10d kube-system kube-apiserver-bs-k8s-master03 1/1 Running 67 10d kube-system kube-controller-manager-bs-k8s-master01 1/1 Running 49 10d kube-system kube-controller-manager-bs-k8s-master02 1/1 Running 38 10d kube-system kube-controller-manager-bs-k8s-master03 1/1 Running 42 10d kube-system kube-proxy-26ffm 1/1 Running 4 10d kube-system kube-proxy-298tr 1/1 Running 5 10d kube-system kube-proxy-hzsmb 1/1 Running 4 10d kube-system kube-proxy-jb4sq 1/1 Running 5 10d kube-system kube-proxy-pt94r 1/1 Running 7 10d kube-system kube-proxy-wljwv 1/1 Running 5 10d kube-system kube-scheduler-bs-k8s-master01 1/1 Running 46 10d kube-system kube-scheduler-bs-k8s-master02 1/1 Running 31 10d kube-system kube-scheduler-bs-k8s-master03 1/1 Running 46 10d kube-system kubernetes-dashboard-887cbd9c6-j7ptq 1/1 Terminating 30 10d kube-system kubernetes-dashboard-887cbd9c6-rzlp4 1/1 Running 7 24h kube-system rbd-provisioner-75b85f85bd-2dclz 1/1 Running 35 3d14h [root@bs-k8s-ceph ~]# ceph osd pool create gitlab 64 pool 'github' created [root@bs-k8s-ceph ~]# cd /etc/ceph/ [root@bs-k8s-ceph ceph]# ceph auth get-or-create client.gitlab mon 'allow r' osd 'allow class-read, allow rwx pool=gitlab' -o ceph.client.gitlab.keyring [root@bs-k8s-node01 ~]# ceph auth get-key client.admin | base64 QVFDNmNVSmV2eU8yRnhBQVBxYzE5Mm5PelNnZk5acmg5aEFQYXc9PQ== [root@bs-k8s-node01 ~]# ceph auth get-key client.gitlab | base64 QVFCUzNFMWVqRDZpTXhBQTgrd3RIajVWb295M1J1QkJDZi9QOXc9PQ== [root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-namespace.yaml namespace/gitlab created [root@bs-k8s-master01 gitlab]# kubectl get namespace gitlab NAME STATUS AGE gitlab Active <invalid> [root@bs-k8s-master01 gitlab]# cat ceph-gitlab-namespace.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-namespace.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: Namespace metadata: name: gitlab [root@bs-k8s-node02 ~]# docker tag quay.io/external_storage/rbd-provisioner:latest harbor.linux.com/kubworker/rbd-provisioner:latest [root@bs-k8s-node02 ~]# docker push harbor.linux.com/kubworker/rbd-provisioner:latest [root@bs-k8s-master01 gitlab]# kubectl apply -f external-storage-rbd-provisioner.yaml serviceaccount/rbd-provisioner created clusterrole.rbac.authorization.k8s.io/rbd-provisioner created clusterrolebinding.rbac.authorization.k8s.io/rbd-provisioner created role.rbac.authorization.k8s.io/rbd-provisioner created rolebinding.rbac.authorization.k8s.io/rbd-provisioner created deployment.apps/rbd-provisioner created [root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlab NAME READY STATUS RESTARTS AGE rbd-provisioner-75b85f85bd-t224w 1/1 Running 0 64s [root@bs-k8s-master01 gitlab]# cat external-storage-rbd-provisioner.yaml apiVersion: v1 kind: ServiceAccount metadata: name: rbd-provisioner namespace: gitlab --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-provisioner rules: - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "update", "patch"] - apiGroups: [""] resources: ["endpoints"] verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["services"] resourceNames: ["kube-dns"] verbs: ["list", "get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-provisioner subjects: - kind: ServiceAccount name: rbd-provisioner namespace: gitlab roleRef: kind: ClusterRole name: rbd-provisioner apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: rbd-provisioner namespace: gitlab rules: - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: rbd-provisioner namespace: gitlab roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: rbd-provisioner subjects: - kind: ServiceAccount name: rbd-provisioner namespace: gitlab --- apiVersion: apps/v1 kind: Deployment metadata: name: rbd-provisioner namespace: gitlab spec: replicas: 1 selector: matchLabels: app: rbd-provisioner strategy: type: Recreate template: metadata: labels: app: rbd-provisioner spec: containers: - name: rbd-provisioner image: "harbor.linux.com/kubworker/rbd-provisioner:latest" env: - name: PROVISIONER_NAME value: ceph.com/rbd serviceAccount: rbd-provisioner imagePullSecrets: - name: login [root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-secret.yaml secret/ceph-gitlab-admin-secret created secret/ceph-gitlab-gitlab-secret created [root@bs-k8s-master01 gitlab]# cat ceph-gitlab-secret.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-secret.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: Secret metadata: name: ceph-gitlab-admin-secret namespace: gitlab data: key: QVFDNmNVSmV2eU8yRnhBQVBxYzE5Mm5PelNnZk5acmg5aEFQYXc9PQ== type: kubernetes.io/rbd --- apiVersion: v1 kind: Secret metadata: name: ceph-gitlab-gitlab-secret namespace: gitlab data: key: QVFCUzNFMWVqRDZpTXhBQTgrd3RIajVWb295M1J1QkJDZi9QOXc9PQ== type: kubernetes.io/rbd [root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-storageclass.yaml storageclass.storage.k8s.io/ceph-gitlab created [root@bs-k8s-master01 gitlab]# cat ceph-gitlab-storageclass.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-storageclass.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ceph-gitlab annotations: storageclass.kubernetes.io/is-default-class: "false" provisioner: ceph.com/rbd reclaimPolicy: Retain parameters: monitors: 20.0.0.206:6789,20.0.0.207:6789,20.0.0.208:6789 adminId: admin adminSecretName: ceph-gitlab-admin-secret adminSecretNamespace: gitlab pool: gitlab fsType: xfs userId: gitlab userSecretName: ceph-gitlab-gitlab-secret imageFormat: "2" imageFeatures: "layering" [root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-gitlab-pvc.yaml persistentvolumeclaim/pvc-ceph-gitlab created [root@bs-k8s-master01 gitlab]# kubectl get pv -n gitlab NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE pvc-494a130d-018c-4be3-9b31-e951cc4367a5 20Gi RWO Retain Bound default/wp-pv-claim ceph-rbd 4d14h pvc-8ffa3182-a2f6-47d9-a71d-ff8e8b379a16 1Gi RWO Retain Bound default/ceph-pvc ceph-rbd 4d16h pvc-ac7d3a09-123e-4614-886c-cded8822a078 20Gi RWO Retain Bound default/mysql-pv-claim ceph-rbd 4d14h pvc-ba59cba5-e394-4b19-8c4b-4018162bbe69 1Gi RWO Retain Bound harbor/pvc-ceph-harbor ceph-harbor 3d14h [root@bs-k8s-master01 gitlab]# kubectl get pvc -n gitlab NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-ceph-gitlab Bound pvc-c7c7aed3-4f8f-4801-a9db-c13f9390af8c 1Gi RWO ceph-gitlab 21s [root@bs-k8s-master01 gitlab]# cat ceph-gitlab-pvc.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-16 #FileName: ceph-harbor-pvc.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-ceph-gitlab namespace: gitlab spec: storageClassName: ceph-gitlab accessModes: - ReadWriteOnce resources: requests: storage: 1Gi [root@bs-k8s-ceph ceph]# ceph osd pool application enable gitlab rbd enabled application 'rbd' on pool 'gitlab' [root@bs-k8s-ceph ceph]# ceph -s cluster: id: 11880418-1a9a-4b55-a353-4b141e2199d8 health: HEALTH_OK services: mon: 3 daemons, quorum bs-hk-hk01,bs-hk-hk02,bs-k8s-ceph mgr: bs-hk-hk02(active), standbys: bs-k8s-ceph, bs-hk-hk01 osd: 6 osds: 6 up, 6 in data: pools: 4 pools, 384 pgs objects: 420 objects, 978 MiB usage: 8.9 GiB used, 105 GiB / 114 GiB avail pgs: 384 active+clean [root@bs-k8s-master01 gitlab]# echo -n "admin" > ./username [root@bs-k8s-master01 gitlab]# echo -n "zisefeizhzu" > ./password [root@bs-k8s-master01 gitlab]# kubectl create secret generic db-user-pass --from-file=./username --from-file=./password secret/db-user-pass created [root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-redis-pvc.yaml persistentvolumeclaim/pvc-ceph-redis created [root@bs-k8s-master01 gitlab]# kubectl get pvc -n gitlab NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-ceph-gitlab Bound pvc-c7c7aed3-4f8f-4801-a9db-c13f9390af8c 1Gi RWO ceph-gitlab 53m pvc-ceph-redis Bound pvc-f8051bd4-4617-4823-b190-3581ec502d7e 1Gi RWO ceph-gitlab 15s [root@bs-k8s-master01 gitlab]# cat ceph-redis-pvc.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-20 #FileName: ceph-redis-pvc.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-ceph-redis namespace: gitlab spec: storageClassName: ceph-gitlab accessModes: - ReadWriteOnce resources: requests: storage: 1Gi [root@bs-k8s-master01 gitlab]# kubectl apply -f gitlab-redis.yaml deployment.apps/redis created service/redis created [root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlab -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES rbd-provisioner-75b85f85bd-t224w 1/1 Running 0 81m 10.209.145.42 bs-k8s-node02 <none> <none> redis-598f9d7ccb-vmrpz 1/1 Running 0 2m7s 10.209.208.22 bs-k8s-node03 <none> <none> [root@bs-k8s-master01 gitlab]# cat gitlab-redis.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-20 #FileName: gitlab-redis.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: apps/v1 kind: Deployment metadata: name: redis namespace: gitlab labels: name: redis spec: selector: matchLabels: name: redis template: metadata: name: redis labels: name: redis spec: containers: - name: redis image: sameersbn/redis imagePullPolicy: IfNotPresent ports: - name: redis containerPort: 6379 volumeMounts: - mountPath: /var/lib/redis name: data livenessProbe: exec: command: - redis-cli - ping initialDelaySeconds: 30 timeoutSeconds: 5 readinessProbe: exec: command: - redis-cli - ping initialDelaySeconds: 5 timeoutSeconds: 1 volumes: - name: data persistentVolumeClaim: claimName: pvc-ceph-redis --- apiVersion: v1 kind: Service metadata: name: redis namespace: gitlab labels: name: redis spec: ports: - name: redis port: 6379 targetPort: redis selector: name: redis [root@bs-k8s-master01 gitlab]# cp ceph-redis-pvc.yaml ceph-postgresql-pvc.yaml [root@bs-k8s-master01 gitlab]# vim ceph-postgresql-pvc.yaml [root@bs-k8s-master01 gitlab]# kubectl apply -f ceph-postgresql-pvc.yaml persistentvolumeclaim/pvc-ceph-postgresql created [root@bs-k8s-master01 gitlab]# kubectl get pvc -n gitlab NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-ceph-gitlab Bound pvc-c7c7aed3-4f8f-4801-a9db-c13f9390af8c 1Gi RWO ceph-gitlab 23h pvc-ceph-postgresql Bound pvc-5995ff51-08fe-4f66-aca8-bda906c44c83 1Gi RWO ceph-gitlab 21h pvc-ceph-redis Bound pvc-f8051bd4-4617-4823-b190-3581ec502d7e 1Gi RWO ceph-gitlab 22h [root@bs-k8s-master01 gitlab]# cat ceph-postgresql-pvc.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-20 #FileName: ceph-redis-pvc.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-ceph-postgresql namespace: gitlab spec: storageClassName: ceph-gitlab accessModes: - ReadWriteOnce resources: requests: storage: 1Gi [root@bs-k8s-master01 gitlab]# kubectl apply -f gitlab-postgresql.yaml deployment.apps/postgresql created service/postgresql created [root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlab NAME READY STATUS RESTARTS AGE postgresql-8c84b7698-w4wpp 1/1 Running 0 5m31s rbd-provisioner-75b85f85bd-t224w 1/1 Running 0 104m redis-598f9d7ccb-vmrpz 1/1 Running 0 24m [root@bs-k8s-master01 gitlab]# cat gitlab-postgresql.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-20 #FileName: gitlab-postgresql.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: apps/v1 kind: Deployment metadata: name: postgresql namespace: gitlab labels: name: postgresql spec: selector: matchLabels: name: postgresql template: metadata: name: postgresql labels: name: postgresql spec: containers: - name: postgresql image: sameersbn/postgresql imagePullPolicy: IfNotPresent env: - name: DB_USER value: gitlab - name: DB_PASS value: zisefeizhu - name: DB_NAME value: gitlab_production - name: DB_EXTENSION value: pg_trgm ports: - name: postgres containerPort: 5432 volumeMounts: - mountPath: /var/lib/postgresql name: data livenessProbe: exec: command: - pg_isready - -h - localhost - -U - postgres initialDelaySeconds: 30 timeoutSeconds: 5 readinessProbe: exec: command: - pg_isready - -h - localhost - -U - postgres initialDelaySeconds: 5 timeoutSeconds: 1 volumes: - name: data persistentVolumeClaim: claimName: pvc-ceph-postgresql --- apiVersion: v1 kind: Service metadata: name: postgresql namespace: gitlab labels: name: postgresql spec: ports: - name: postgres port: 5432 targetPort: postgres selector: name: postgresql [root@bs-k8s-master01 gitlab]# kubectl apply -f gitlab-gitlab.yaml deployment.apps/gitlab created service/gitlab created ingress.extensions/gitlab created [root@bs-k8s-master01 gitlab]# kubectl get ingress -n gitlab NAME HOSTS ADDRESS PORTS AGE gitlab gitlab.linux.com 80 106s [root@bs-k8s-master01 gitlab]# kubectl get service -n gitlab NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE gitlab NodePort 10.96.120.85 <none> 80:32208/TCP,22:30022/TCP 2m33s postgresql ClusterIP 10.111.30.80 <none> 5432/TCP 41m redis ClusterIP 10.103.127.83 <none> 6379/TCP [root@bs-k8s-master01 gitlab]# kubectl get pods -n gitlab -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES gitlab-6468b9f55c-mzcmj 1/1 Running 1 4m59s 10.209.208.41 bs-k8s-node03 <none> <none> postgresql-8c84b7698-w4wpp 1/1 Running 0 101m 10.209.145.43 bs-k8s-node02 <none> <none> rbd-provisioner-75b85f85bd-t224w 1/1 Running 0 3h20m 10.209.145.42 bs-k8s-node02 <none> <none> redis-598f9d7ccb-vmrpz 1/1 Running 0 120m 10.209.208.22 bs-k8s-node03 <none> [root@bs-k8s-master01 gitlab]# cat gitlab-gitlab.yaml ########################################################################## #Author: zisefeizhu #QQ: 2********0 #Date: 2020-02-20 #FileName: gitlab-gitlab.yaml #URL: https://www.cnblogs.com/zisefeizhu/ #Description: The test script #Copyright (C): 2020 All rights reserved ########################################################################### apiVersion: apps/v1 kind: Deployment metadata: name: gitlab namespace: gitlab labels: name: gitlab spec: selector: matchLabels: name: gitlab template: metadata: name: gitlab labels: name: gitlab spec: containers: - name: gitlab image: sameersbn/gitlab:12.1.6 imagePullPolicy: IfNotPresent env: - name: TZ value: Asia/Shanghai - name: GITLAB_TIMEZONE value: Beijing - name: GITLAB_SECRETS_DB_KEY_BASE value: long-and-random-alpha-numeric-string - name: GITLAB_SECRETS_SECRET_KEY_BASE value: long-and-random-alpha-numeric-string - name: GITLAB_SECRETS_OTP_KEY_BASE value: long-and-random-alpha-numeric-string - name: GITLAB_ROOT_PASSWORD value: zisefeizhu - name: GITLAB_ROOT_EMAIL value: 2350835860@qq.com - name: GITLAB_HOST value: gitlab.linux.com - name: GITLAB_PORT value: "80" - name: GITLAB_SSH_PORT value: "30022" - name: GITLAB_NOTIFY_ON_BROKEN_BUILDS value: "true" - name: GITLAB_NOTIFY_PUSHER value: "false" - name: GITLAB_BACKUP_SCHEDULE value: daily - name: GITLAB_BACKUP_TIME value: 01:00 - name: DB_TYPE value: postgres - name: DB_HOST value: postgresql - name: DB_PORT value: "5432" - name: DB_USER value: gitlab - name: DB_PASS value: zisefeizhu - name: DB_NAME value: gitlab_production - name: REDIS_HOST value: redis - name: REDIS_PORT value: "6379" ports: - name: http containerPort: 80 - name: ssh containerPort: 22 volumeMounts: - mountPath: /home/git/data name: data livenessProbe: httpGet: path: / port: 80 initialDelaySeconds: 180 timeoutSeconds: 5 readinessProbe: httpGet: path: / port: 80 initialDelaySeconds: 5 timeoutSeconds: 1 volumes: - name: data persistentVolumeClaim: claimName: pvc-ceph-gitlab --- apiVersion: v1 kind: Service metadata: name: gitlab namespace: gitlab labels: name: gitlab spec: ports: - name: http port: 80 targetPort: http - name: ssh port: 22 targetPort: ssh nodePort: 30022 type: NodePort selector: name: gitlab --- apiVersion: extensions/v1beta1 kind: Ingress metadata: name: gitlab namespace: gitlab annotations: kubernetes.io/ingress.class: traefik spec: rules: - host: gitlab.linux.com http: paths: - backend: serviceName: gitlab servicePort: http