root@ubuntu:~# kubectl apply -f job.yaml
job.batch.volcano.sh/test-job created
先触发AddPodGroupV1beta1后AddPod
AddPod
其中 AddPod 会干以下几件事：
- 对 pod 生成一个对应的 taskInfo，并生成一个 jobID。
- 如果在 schedulerCache 的 jobs 没有这个 jobID，创建这个 jobInfo，然后把这个 task 放到 job.Tasks 中。
- 如果已经分配了 node，也会在 schedulerCache的 nodes 里面记录这个task。
pod 和taskinfo
// Assumes that lock is already acquired. func (sc *SchedulerCache) addPod(pod *v1.Pod) error { pi := schedulingapi.NewTaskInfo(pod) return sc.addTask(pi) }
(dlv) b AddPod Command failed: Location "AddPod" ambiguous: volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod, k8s.io/kubernetes/pkg/scheduler/nodeinfo.(*NodeInfo).AddPod, k8s.io/kubernetes/pkg/scheduler/framework/plugins/interpodaffinity.(*InterPodAffinity).AddPod… (dlv) b cache.(*SchedulerCache).AddPod Breakpoint 2 (enabled) set at 0xea9920 for volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:179 (dlv) b AddPodGroupV1beta1 Breakpoint 3 (enabled) set at 0xeab240 for volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 (dlv) c > volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 (hits goroutine(289):1 total:1) (PC: 0xeab240) Warning: debugging optimized function (dlv) bt 0 0x0000000000eab240 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1 at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 1 0x0000000000eaeb68 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1-fm at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 2 0x0000000000c9ccc8 in k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:218 3 0x0000000000c9aed8 in k8s.io/client-go/tools/cache.(*processorListener).run.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:744 4 0x0000000000339ae4 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 5 0x0000000000338b54 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 6 0x0000000000338ac8 in k8s.io/apimachinery/pkg/util/wait.JitterUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 7 0x0000000000c96220 in k8s.io/apimachinery/pkg/util/wait.Until at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 8 0x0000000000c96220 in k8s.io/client-go/tools/cache.(*processorListener).run at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:738 9 0x0000000000c9c8c8 in k8s.io/client-go/tools/cache.(*processorListener).run-fm at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:732 10 0x0000000000339a44 in k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 11 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) p obj interface {}(*volcano.sh/volcano/pkg/apis/scheduling/v1beta1.PodGroup) *{ TypeMeta: k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta {Kind: "", APIVersion: ""}, ObjectMeta: k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta { Name: "test-job", GenerateName: "", Namespace: "default", SelfLink: "/apis/scheduling.volcano.sh/v1beta1/namespaces/default/podgroups...+9 more", UID: "513b55a7-d2f5-4ce4-9598-8057258eb86f", ResourceVersion: "3023789", Generation: 1, CreationTimestamp: (*"k8s.io/apimachinery/pkg/apis/meta/v1.Time")(0x4000b59d08), DeletionTimestamp: *k8s.io/apimachinery/pkg/apis/meta/v1.Time nil, DeletionGracePeriodSeconds: *int64 nil, Labels: map[string]string nil, Annotations: map[string]string [...], OwnerReferences: []k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference")(0x4000baa140), ], Finalizers: []string len: 0, cap: 0, nil, ClusterName: "", ManagedFields: []k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry")(0x4000baa0f0), ],}, Spec: volcano.sh/volcano/pkg/apis/scheduling/v1beta1.PodGroupSpec { MinMember: 3, Queue: "default", PriorityClassName: "", MinResources: *k8s.io/api/core/v1.ResourceList [...],}, Status: volcano.sh/volcano/pkg/apis/scheduling/v1beta1.PodGroupStatus { Phase: "", Conditions: []volcano.sh/volcano/pkg/apis/scheduling/v1beta1.PodGroupCondition len: 0, cap: 0, nil, Running: 0, Succeeded: 0, Failed: 0,},} (dlv) c > volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:179 (hits goroutine(174):1 total:1) (PC: 0xea9920) Warning: debugging optimized function (dlv) bt 0 0x0000000000ea9920 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:179 1 0x0000000000eae868 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod-fm at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:179 2 0x0000000000c9ccc8 in k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:218 3 0x0000000000c8ab54 in k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:250 4 0x0000000000c9ca18 in k8s.io/client-go/tools/cache.(*FilteringResourceEventHandler).OnAdd at <autogenerated>:1 5 0x0000000000c9aed8 in k8s.io/client-go/tools/cache.(*processorListener).run.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:744 6 0x0000000000339ae4 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 7 0x0000000000338b54 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 8 0x0000000000338ac8 in k8s.io/apimachinery/pkg/util/wait.JitterUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 9 0x0000000000c96220 in k8s.io/apimachinery/pkg/util/wait.Until at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 10 0x0000000000c96220 in k8s.io/client-go/tools/cache.(*processorListener).run at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:738 11 0x0000000000c9c8c8 in k8s.io/client-go/tools/cache.(*processorListener).run-fm at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:732 12 0x0000000000339a44 in k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 13 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) p obj interface {}(*k8s.io/api/core/v1.Pod) *{ TypeMeta: k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta {Kind: "", APIVersion: ""}, ObjectMeta: k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta { Name: "test-job-default-nginx-4", GenerateName: "", Namespace: "default", SelfLink: "/api/v1/namespaces/default/pods/test-job-default-nginx-4", UID: "f7871ad6-f9ac-44d9-8e78-ecb3ece1bfa2", ResourceVersion: "3024173", Generation: 0, CreationTimestamp: (*"k8s.io/apimachinery/pkg/apis/meta/v1.Time")(0x4000686c88), DeletionTimestamp: *k8s.io/apimachinery/pkg/apis/meta/v1.Time nil, DeletionGracePeriodSeconds: *int64 nil, Labels: map[string]string [...], Annotations: map[string]string [...], OwnerReferences: []k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference")(0x400081aa00), ], Finalizers: []string len: 0, cap: 0, nil, ClusterName: "", ManagedFields: []k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry")(0x400081aaa0), ],}, Spec: k8s.io/api/core/v1.PodSpec { Volumes: []k8s.io/api/core/v1.Volume len: 3, cap: 4, [ (*"k8s.io/api/core/v1.Volume")(0x4000687000), (*"k8s.io/api/core/v1.Volume")(0x40006870f0), (*"k8s.io/api/core/v1.Volume")(0x40006871e0), ], InitContainers: []k8s.io/api/core/v1.Container len: 0, cap: 0, nil, Containers: []k8s.io/api/core/v1.Container len: 1, cap: 1, [ (*"k8s.io/api/core/v1.Container")(0x4000512000), ], EphemeralContainers: []k8s.io/api/core/v1.EphemeralContainer len: 0, cap: 0, nil, RestartPolicy: "OnFailure", TerminationGracePeriodSeconds: *30, ActiveDeadlineSeconds: *int64 nil, DNSPolicy: "ClusterFirst", NodeSelector: map[string]string nil, ServiceAccountName: "default", DeprecatedServiceAccount: "default", AutomountServiceAccountToken: *bool nil, NodeName: "", HostNetwork: false, HostPID: false, HostIPC: false, ShareProcessNamespace: *bool nil, SecurityContext: *(*"k8s.io/api/core/v1.PodSecurityContext")(0x400026e540), ImagePullSecrets: []k8s.io/api/core/v1.LocalObjectReference len: 0, cap: 0, nil, Hostname: "test-job-default-nginx-4", Subdomain: "test-job", Affinity: *k8s.io/api/core/v1.Affinity nil, SchedulerName: "volcano", Tolerations: []k8s.io/api/core/v1.Toleration len: 2, cap: 2, [ (*"k8s.io/api/core/v1.Toleration")(0x40008da2d0), (*"k8s.io/api/core/v1.Toleration")(0x40008da318), ], HostAliases: []k8s.io/api/core/v1.HostAlias len: 0, cap: 0, nil, PriorityClassName: "", Priority: *0, DNSConfig: *k8s.io/api/core/v1.PodDNSConfig nil, ReadinessGates: []k8s.io/api/core/v1.PodReadinessGate len: 0, cap: 0, nil, RuntimeClassName: *string nil, EnableServiceLinks: *true, PreemptionPolicy: *k8s.io/api/core/v1.PreemptionPolicy nil, Overhead: k8s.io/api/core/v1.ResourceList nil, TopologySpreadConstraints: []k8s.io/api/core/v1.TopologySpreadConstraint len: 0, cap: 0, nil,}, Status: k8s.io/api/core/v1.PodStatus { Phase: "Pending", Conditions: []k8s.io/api/core/v1.PodCondition len: 0, cap: 0, nil, Message: "", Reason: "", NominatedNodeName: "", HostIP: "", PodIP: "", PodIPs: []k8s.io/api/core/v1.PodIP len: 0, cap: 0, nil, StartTime: *k8s.io/apimachinery/pkg/apis/meta/v1.Time nil, InitContainerStatuses: []k8s.io/api/core/v1.ContainerStatus len: 0, cap: 0, nil, ContainerStatuses: []k8s.io/api/core/v1.ContainerStatus len: 0, cap: 0, nil, QOSClass: "Burstable", EphemeralContainerStatuses: []k8s.io/api/core/v1.ContainerStatus len: 0, cap: 0, nil,},} (dlv) b getOrCreateJob Breakpoint 4 (enabled) set at 0xea8a20 for volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).getOrCreateJob() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:43 (dlv) c > volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).getOrCreateJob() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:43 (hits goroutine(174):1 total:1) (PC: 0xea8a20) Warning: debugging optimized function (dlv) bt 0 0x0000000000ea8a20 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).getOrCreateJob at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:43 1 0x0000000000ea8cc0 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).addTask at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:60 2 0x0000000000ea8e54 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).addPod at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:83 3 0x0000000000ea99a4 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:189 4 0x0000000000eae868 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod-fm at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:179 5 0x0000000000c9ccc8 in k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:218 6 0x0000000000c8ab54 in k8s.io/client-go/tools/cache.FilteringResourceEventHandler.OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:250 7 0x0000000000c9ca18 in k8s.io/client-go/tools/cache.(*FilteringResourceEventHandler).OnAdd at <autogenerated>:1 8 0x0000000000c9aed8 in k8s.io/client-go/tools/cache.(*processorListener).run.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:744 9 0x0000000000339ae4 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 10 0x0000000000338b54 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 11 0x0000000000338ac8 in k8s.io/apimachinery/pkg/util/wait.JitterUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 12 0x0000000000c96220 in k8s.io/apimachinery/pkg/util/wait.Until at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 13 0x0000000000c96220 in k8s.io/client-go/tools/cache.(*processorListener).run at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:738 14 0x0000000000c9c8c8 in k8s.io/client-go/tools/cache.(*processorListener).run-fm at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:732 15 0x0000000000339a44 in k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 16 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) p sc.Jobs[pi.Job] *volcano.sh/volcano/pkg/scheduler/api.JobInfo { UID: "default/test-job", Name: "test-job", Namespace: "default", Queue: "default", Priority: 0, MinAvailable: 3, NodesFitDelta: volcano.sh/volcano/pkg/scheduler/api.NodeResourceMap [], JobFitErrors: "", NodesFitErrors: map[volcano.sh/volcano/pkg/scheduler/api.TaskID]*volcano.sh/volcano/pkg/scheduler/api.FitErrors [], TaskStatusIndex: map[volcano.sh/volcano/pkg/scheduler/api.TaskStatus]volcano.sh/volcano/pkg/scheduler/api.tasksMap [], Tasks: volcano.sh/volcano/pkg/scheduler/api.tasksMap [], Allocated: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 0, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, TotalRequest: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 0, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, CreationTimestamp: k8s.io/apimachinery/pkg/apis/meta/v1.Time { Time: (*time.Time)(0x4000306138),}, PodGroup: *volcano.sh/volcano/pkg/scheduler/api.PodGroup { PodGroup: (*"volcano.sh/volcano/pkg/apis/scheduling.PodGroup")(0x400020a340), Version: "v1beta1",},} (dlv) p sc.Jobs[pi.Job].PodGroup *volcano.sh/volcano/pkg/scheduler/api.PodGroup { PodGroup: volcano.sh/volcano/pkg/apis/scheduling.PodGroup { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x400020a340), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x400020a360), Spec: (*"volcano.sh/volcano/pkg/apis/scheduling.PodGroupSpec")(0x400020a458), Status: (*"volcano.sh/volcano/pkg/apis/scheduling.PodGroupStatus")(0x400020a488),}, Version: "v1beta1",} (dlv) p sc.Jobs[pi.Job].PodGroup.ObjectMeta k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta { Name: "test-job", GenerateName: "", Namespace: "default", SelfLink: "/apis/scheduling.volcano.sh/v1beta1/namespaces/default/podgroups...+9 more", UID: "513b55a7-d2f5-4ce4-9598-8057258eb86f", ResourceVersion: "3024170", Generation: 3, CreationTimestamp: k8s.io/apimachinery/pkg/apis/meta/v1.Time { Time: (*time.Time)(0x400020a3c8),}, DeletionTimestamp: *k8s.io/apimachinery/pkg/apis/meta/v1.Time nil, DeletionGracePeriodSeconds: *int64 nil, Labels: map[string]string nil, Annotations: map[string]string [ "kubectl.kubernetes.io/last-applied-configuration": "{"apiVersion":"batch.volcano.sh/v1alpha1","kind":"Job","metadata...+484 more", ], OwnerReferences: []k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference")(0x400081a640), ], Finalizers: []string len: 0, cap: 0, nil, ClusterName: "", ManagedFields: []k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry len: 2, cap: 2, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry")(0x4000229180), (*"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry")(0x40002291d0), ],} (dlv)
(dlv) c > volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPod() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:179 (hits goroutine(174):2 total:2) (PC: 0xea9920) Warning: debugging optimized function (dlv) p obj interface {}(*k8s.io/api/core/v1.Pod) *{ TypeMeta: k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta {Kind: "", APIVersion: ""}, ObjectMeta: k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta { Name: "test-job-default-nginx-3", GenerateName: "", Namespace: "default", SelfLink: "/api/v1/namespaces/default/pods/test-job-default-nginx-3", UID: "a1a6c98d-6c8c-4ac7-94be-b739068c9f20", ResourceVersion: "3024174", Generation: 0, CreationTimestamp: (*"k8s.io/apimachinery/pkg/apis/meta/v1.Time")(0x4000687488), DeletionTimestamp: *k8s.io/apimachinery/pkg/apis/meta/v1.Time nil, DeletionGracePeriodSeconds: *int64 nil, Labels: map[string]string [...], Annotations: map[string]string [...], OwnerReferences: []k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference")(0x400081ad20), ], Finalizers: []string len: 0, cap: 0, nil, ClusterName: "", ManagedFields: []k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry")(0x400081ae60), ],}, Spec: k8s.io/api/core/v1.PodSpec { Volumes: []k8s.io/api/core/v1.Volume len: 3, cap: 4, [ (*"k8s.io/api/core/v1.Volume")(0x4000a5e000), (*"k8s.io/api/core/v1.Volume")(0x4000a5e0f0), (*"k8s.io/api/core/v1.Volume")(0x4000a5e1e0), ],
job什么时候创建的
如果是触发了AddPodGroupV1beta1 会做以下几件事：
- 也会找schedulerCache jobs 里面有没有这个 podgroup 对应的 job，没有就创建一下。
- 然后把 podgroup 的信息（MinAvailable，Queue）等传递给 JobInfo
可以认为看到pod，podgroup 的创建，eventHandler 会在 schedulerCache 的 jobs 增加一个对应 jobInfo。
(dlv) bt 0 0x0000000000e9a620 in volcano.sh/volcano/pkg/scheduler/api.NewJobInfo at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:154 1 0x0000000000eab088 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).setPodGroup at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:359 2 0x0000000000eab418 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1 at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:412 3 0x0000000000eaeb68 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1-fm at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 4 0x0000000000c9ccc8 in k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:218 5 0x0000000000c9aed8 in k8s.io/client-go/tools/cache.(*processorListener).run.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:744 6 0x0000000000339ae4 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 7 0x0000000000338b54 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 8 0x0000000000338ac8 in k8s.io/apimachinery/pkg/util/wait.JitterUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 9 0x0000000000c96220 in k8s.io/apimachinery/pkg/util/wait.Until at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 10 0x0000000000c96220 in k8s.io/client-go/tools/cache.(*processorListener).run at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:738 11 0x0000000000c9c8c8 in k8s.io/client-go/tools/cache.(*processorListener).run-fm at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:732 12 0x0000000000339a44 in k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 13 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148
type JobInfo struct { UID JobID Name string Namespace string Queue QueueID Priority int32 MinAvailable int32 JobFitErrors string NodesFitErrors map[TaskID]*FitErrors // All tasks of the Job. TaskStatusIndex map[TaskStatus]tasksMap Tasks tasksMap Allocated *Resource TotalRequest *Resource CreationTimestamp metav1.Time PodGroup *PodGroup ScheduleStartTimestamp metav1.Time } type tasksMap map[TaskID]*TaskInfo type TaskInfo struct { UID TaskID Job JobID Name string Namespace string // Resreq is the resource that used when task running. Resreq *Resource // InitResreq is the resource that used to launch a task. InitResreq *Resource NodeName string Status TaskStatus Priority int32 VolumeReady bool Pod *v1.Pod }
可以看到 JobInfo 主要有一个 map[TaskID]*TaskInfo 的 taskmap 记录工作负载信息，同时记录了 podgroup 和 queue 的信息方便相互查询。TaskInfo 是一个pod 信息的封装，主要添加了 JobID，保证可以通过 task 找到所属的 jobInfo。下面 NodeInfo 会通过 tasks 记录在 node 上运行的工作负载，同时记录不同资源的使用情况。queueInfo 则是记录队列资源（queue）的状态。
root@ubuntu:~# kubectl apply -f job.yaml job.batch.volcano.sh/test-job created root@ubuntu:~# kubectl get jobs No resources found in default namespace. root@ubuntu:~# kubectl get pods NAME READY STATUS RESTARTS AGE example-foo-54dc4db9fc-lqz9j 1/1 Running 0 4d19h job-1-nginx-0 0/1 Completed 0 7d4h root@ubuntu:~#
root@ubuntu:~# cat job.yaml apiVersion: batch.volcano.sh/v1alpha1 kind: Job metadata: name: test-job spec: minAvailable: 3 schedulerName: volcano policies: - event: PodEvicted action: RestartJob plugins: ssh: [] env: [] svc: [] maxRetry: 5 queue: default # Comment out the following section to enable volumes for job input/output. #volumes: # - mountPath: "/myinput" # - mountPath: "/myoutput" # volumeClaimName: "testvolumeclaimname" # volumeClaim: # accessModes: [ "ReadWriteOnce" ] # storageClassName: "my-storage-class" # resources: # requests: # storage: 1Gi tasks: - replicas: 6 name: "default-nginx" template: metadata: name: web spec: containers: - image: nginx imagePullPolicy: IfNotPresent name: nginx resources: requests: cpu: "1" restartPolicy: OnFailure
/ NewJobInfo creates a new jobInfo for set of tasks func NewJobInfo(uid JobID, tasks ...*TaskInfo) *JobInfo { job := &JobInfo{ UID: uid, MinAvailable: 0, NodesFitErrors: make(map[TaskID]*FitErrors), Allocated: EmptyResource(), TotalRequest: EmptyResource(), TaskStatusIndex: map[TaskStatus]tasksMap{}, Tasks: tasksMap{}, TaskMinAvailable: map[TaskID]int32{}, } for _, task := range tasks { job.AddTaskInfo(task) } return job }
(dlv) b cache.(*SchedulerCache).AddPodGroupV1beta1 Breakpoint 1 (enabled) set at 0xeab240 for volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 (dlv) b NewJobInfo Breakpoint 2 (enabled) set at 0xe9a620 for volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:154 (dlv) c > volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 (hits goroutine(289):1 total:1) (PC: 0xeab240) Warning: debugging optimized function (dlv) p sc *volcano.sh/volcano/pkg/scheduler/cache.SchedulerCache { Mutex: sync.Mutex {state: 0, sema: 0}, kubeClient: *k8s.io/client-go/kubernetes.Clientset { DiscoveryClient: *(*"k8s.io/client-go/discovery.DiscoveryClient")(0x40005761c0), admissionregistrationV1: *(*"k8s.io/client-go/kubernetes/typed/admissionregistration/v1.AdmissionregistrationV1Client")(0x400050f380), admissionregistrationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1.AdmissionregistrationV1beta1Client")(0x400050f3e0), appsV1: *(*"k8s.io/client-go/kubernetes/typed/apps/v1.AppsV1Client")(0x400050f440), appsV1beta1: *(*"k8s.io/client-go/kubernetes/typed/apps/v1beta1.AppsV1beta1Client")(0x400050f4a0), appsV1beta2: *(*"k8s.io/client-go/kubernetes/typed/apps/v1beta2.AppsV1beta2Client")(0x400050f500), auditregistrationV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1.AuditregistrationV1alpha1Client")(0x400050f560), authenticationV1: *(*"k8s.io/client-go/kubernetes/typed/authentication/v1.AuthenticationV1Client")(0x400050f5c0), authenticationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/authentication/v1beta1.AuthenticationV1beta1Client")(0x400050f620), authorizationV1: *(*"k8s.io/client-go/kubernetes/typed/authorization/v1.AuthorizationV1Client")(0x400050f680), authorizationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/authorization/v1beta1.AuthorizationV1beta1Client")(0x400050f6e0), autoscalingV1: *(*"k8s.io/client-go/kubernetes/typed/autoscaling/v1.AutoscalingV1Client")(0x400050f740), autoscalingV2beta1: *(*"k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1.AutoscalingV2beta1Client")(0x400050f7a0), autoscalingV2beta2: *(*"k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2.AutoscalingV2beta2Client")(0x400050f800), batchV1: *(*"k8s.io/client-go/kubernetes/typed/batch/v1.BatchV1Client")(0x400050f860), batchV1beta1: *(*"k8s.io/client-go/kubernetes/typed/batch/v1beta1.BatchV1beta1Client")(0x400050f8c0), batchV2alpha1: *(*"k8s.io/client-go/kubernetes/typed/batch/v2alpha1.BatchV2alpha1Client")(0x400050f920), certificatesV1beta1: *(*"k8s.io/client-go/kubernetes/typed/certificates/v1beta1.CertificatesV1beta1Client")(0x400050f980), coordinationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/coordination/v1beta1.CoordinationV1beta1Client")(0x400050f9e0), coordinationV1: *(*"k8s.io/client-go/kubernetes/typed/coordination/v1.CoordinationV1Client")(0x400050fa40), coreV1: *(*"k8s.io/client-go/kubernetes/typed/core/v1.CoreV1Client")(0x400050faa0), discoveryV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/discovery/v1alpha1.DiscoveryV1alpha1Client")(0x400050fb00), discoveryV1beta1: *(*"k8s.io/client-go/kubernetes/typed/discovery/v1beta1.DiscoveryV1beta1Client")(0x400050fb60), eventsV1beta1: *(*"k8s.io/client-go/kubernetes/typed/events/v1beta1.EventsV1beta1Client")(0x400050fbc0), extensionsV1beta1: *(*"k8s.io/client-go/kubernetes/typed/extensions/v1beta1.ExtensionsV1beta1Client")(0x400050fc20), flowcontrolV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1.FlowcontrolV1alpha1Client")(0x400050fc80), networkingV1: *(*"k8s.io/client-go/kubernetes/typed/networking/v1.NetworkingV1Client")(0x400050fce0), networkingV1beta1: *(*"k8s.io/client-go/kubernetes/typed/networking/v1beta1.NetworkingV1beta1Client")(0x400050fd40), nodeV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/node/v1alpha1.NodeV1alpha1Client")(0x400050fda0), nodeV1beta1: *(*"k8s.io/client-go/kubernetes/typed/node/v1beta1.NodeV1beta1Client")(0x400050fe00), policyV1beta1: *(*"k8s.io/client-go/kubernetes/typed/policy/v1beta1.PolicyV1beta1Client")(0x400050fe60), rbacV1: *(*"k8s.io/client-go/kubernetes/typed/rbac/v1.RbacV1Client")(0x400050fec0), rbacV1beta1: *(*"k8s.io/client-go/kubernetes/typed/rbac/v1beta1.RbacV1beta1Client")(0x400050ff20), rbacV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/rbac/v1alpha1.RbacV1alpha1Client")(0x400050ff80), schedulingV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1.SchedulingV1alpha1Client")(0x400050ffe0), schedulingV1beta1: *(*"k8s.io/client-go/kubernetes/typed/scheduling/v1beta1.SchedulingV1beta1Client")(0x400056e040), schedulingV1: *(*"k8s.io/client-go/kubernetes/typed/scheduling/v1.SchedulingV1Client")(0x400056e0a0), settingsV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/settings/v1alpha1.SettingsV1alpha1Client")(0x400056e100), storageV1beta1: *(*"k8s.io/client-go/kubernetes/typed/storage/v1beta1.StorageV1beta1Client")(0x400056e160), storageV1: *(*"k8s.io/client-go/kubernetes/typed/storage/v1.StorageV1Client")(0x400056e1c0), storageV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/storage/v1alpha1.StorageV1alpha1Client")(0x400056e220),}, vcClient: *volcano.sh/volcano/pkg/client/clientset/versioned.Clientset { DiscoveryClient: *(*"k8s.io/client-go/discovery.DiscoveryClient")(0x4000576480), batchV1alpha1: *(*"volcano.sh/volcano/pkg/client/clientset/versioned/typed/batch/v1alpha1.BatchV1alpha1Client")(0x400056e330), busV1alpha1: *(*"volcano.sh/volcano/pkg/client/clientset/versioned/typed/bus/v1alpha1.BusV1alpha1Client")(0x400056e390), schedulingV1beta1: *(*"volcano.sh/volcano/pkg/client/clientset/versioned/typed/scheduling/v1beta1.SchedulingV1beta1Client")(0x400056e3f0),}, defaultQueue: "default", schedulerName: "volcano", podInformer: k8s.io/client-go/informers/core/v1.PodInformer(*k8s.io/client-go/informers/core/v1.podInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, nodeInformer: k8s.io/client-go/informers/core/v1.NodeInformer(*k8s.io/client-go/informers/core/v1.nodeInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, podGroupInformerV1beta1: volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.PodGroupInformer(*volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.podGroupInformer) *{ factory: volcano.sh/volcano/pkg/client/informers/externalversions/internalinterfaces.SharedInformerFactory(*volcano.sh/volcano/pkg/client/informers/externalversions.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, queueInformerV1beta1: volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.QueueInformer(*volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.queueInformer) *{ factory: volcano.sh/volcano/pkg/client/informers/externalversions/internalinterfaces.SharedInformerFactory(*volcano.sh/volcano/pkg/client/informers/externalversions.sharedInformerFactory) ..., tweakListOptions: nil,}, pvInformer: k8s.io/client-go/informers/core/v1.PersistentVolumeInformer(*k8s.io/client-go/informers/core/v1.persistentVolumeInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, pvcInformer: k8s.io/client-go/informers/core/v1.PersistentVolumeClaimInformer(*k8s.io/client-go/informers/core/v1.persistentVolumeClaimInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, scInformer: k8s.io/client-go/informers/storage/v1.StorageClassInformer(*k8s.io/client-go/informers/storage/v1.storageClassInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, pcInformer: k8s.io/client-go/informers/scheduling/v1beta1.PriorityClassInformer(*k8s.io/client-go/informers/scheduling/v1beta1.priorityClassInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, quotaInformer: k8s.io/client-go/informers/core/v1.ResourceQuotaInformer(*k8s.io/client-go/informers/core/v1.resourceQuotaInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, Binder: volcano.sh/volcano/pkg/scheduler/cache.Binder(*volcano.sh/volcano/pkg/scheduler/cache.defaultBinder) *{ kubeclient: *(*"k8s.io/client-go/kubernetes.Clientset")(0x4000548160),}, Evictor: volcano.sh/volcano/pkg/scheduler/cache.Evictor(*volcano.sh/volcano/pkg/scheduler/cache.defaultEvictor) *{ kubeclient: *(*"k8s.io/client-go/kubernetes.Clientset")(0x4000548160), recorder: k8s.io/client-go/tools/record.EventRecorder(*k8s.io/client-go/tools/record.recorderImpl) ...,}, StatusUpdater: volcano.sh/volcano/pkg/scheduler/cache.StatusUpdater(*volcano.sh/volcano/pkg/scheduler/cache.defaultStatusUpdater) *{ kubeclient: *(*"k8s.io/client-go/kubernetes.Clientset")(0x4000548160), vcclient: *(*"volcano.sh/volcano/pkg/client/clientset/versioned.Clientset")(0x4000576200),}, VolumeBinder: volcano.sh/volcano/pkg/scheduler/cache.VolumeBinder(*volcano.sh/volcano/pkg/scheduler/cache.defaultVolumeBinder) *{ volumeBinder: k8s.io/kubernetes/pkg/controller/volume/scheduling.SchedulerVolumeBinder(*k8s.io/kubernetes/pkg/controller/volume/scheduling.volumeBinder) ...,}, Recorder: k8s.io/client-go/tools/record.EventRecorder(*k8s.io/client-go/tools/record.recorderImpl) *{ scheme: *(*"k8s.io/apimachinery/pkg/runtime.Scheme")(0x4000509110), source: (*"k8s.io/api/core/v1.EventSource")(0x400013a8c8), Broadcaster: *(*"k8s.io/apimachinery/pkg/watch.Broadcaster")(0x400013a840), clock: k8s.io/apimachinery/pkg/util/clock.Clock(k8s.io/apimachinery/pkg/util/clock.RealClock) *(*"k8s.io/apimachinery/pkg/util/clock.Clock")(0x400013a8f0),}, Jobs: map[volcano.sh/volcano/pkg/scheduler/api.JobID]*volcano.sh/volcano/pkg/scheduler/api.JobInfo [ "default/job-1": *(*"volcano.sh/volcano/pkg/scheduler/api.JobInfo")(0x4000666000), ], Nodes: map[string]*volcano.sh/volcano/pkg/scheduler/api.NodeInfo [ "bogon": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654880), "cloud": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654900), "centos7": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654980), "ubuntu": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654a00), ], Queues: map[volcano.sh/volcano/pkg/scheduler/api.QueueID]*volcano.sh/volcano/pkg/scheduler/api.QueueInfo [ "default": *(*"volcano.sh/volcano/pkg/scheduler/api.QueueInfo")(0x400090cf60), "test": *(*"volcano.sh/volcano/pkg/scheduler/api.QueueInfo")(0x4000721680), ], PriorityClasses: map[string]*k8s.io/api/scheduling/v1beta1.PriorityClass [ "system-node-critical": *(*"k8s.io/api/scheduling/v1beta1.PriorityClass")(0x4000324a00), "system-cluster-critical": *(*"k8s.io/api/scheduling/v1beta1.PriorityClass")(0x4000324b38), ], defaultPriorityClass: *k8s.io/api/scheduling/v1beta1.PriorityClass nil, defaultPriority: 0, NamespaceCollection: map[string]*volcano.sh/volcano/pkg/scheduler/api.NamespaceCollection [], errTasks: k8s.io/client-go/util/workqueue.RateLimitingInterface(*k8s.io/client-go/util/workqueue.rateLimitingType) *{ DelayingInterface: k8s.io/client-go/util/workqueue.DelayingInterface(*k8s.io/client-go/util/workqueue.delayingType) ..., rateLimiter: k8s.io/client-go/util/workqueue.RateLimiter(*k8s.io/client-go/util/workqueue.MaxOfRateLimiter) ...,}, deletedJobs: k8s.io/client-go/util/workqueue.RateLimitingInterface(*k8s.io/client-go/util/workqueue.rateLimitingType) *{ DelayingInterface: k8s.io/client-go/util/workqueue.DelayingInterface(*k8s.io/client-go/util/workqueue.delayingType) ..., rateLimiter: k8s.io/client-go/util/workqueue.RateLimiter(*k8s.io/client-go/util/workqueue.MaxOfRateLimiter) ...,},} (dlv) p sc.Jobs map[volcano.sh/volcano/pkg/scheduler/api.JobID]*volcano.sh/volcano/pkg/scheduler/api.JobInfo [ "default/job-1": *{ UID: "default/job-1", Name: "", Namespace: "", Queue: "", Priority: 0, MinAvailable: 0, NodesFitDelta: volcano.sh/volcano/pkg/scheduler/api.NodeResourceMap [], JobFitErrors: "", NodesFitErrors: map[volcano.sh/volcano/pkg/scheduler/api.TaskID]*volcano.sh/volcano/pkg/scheduler/api.FitErrors [], TaskStatusIndex: map[volcano.sh/volcano/pkg/scheduler/api.TaskStatus]volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Tasks: volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Allocated: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962e80), TotalRequest: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962ea0), CreationTimestamp: (*"k8s.io/apimachinery/pkg/apis/meta/v1.Time")(0x4000666088), PodGroup: *volcano.sh/volcano/pkg/scheduler/api.PodGroup nil,}, ] (dlv) map[volcano.sh/volcano/pkg/scheduler/api.JobID]*volcano.sh/volcano/pkg/scheduler/api.JobInfo [ "default/job-1": *{ UID: "default/job-1", Name: "", Namespace: "", Queue: "", Priority: 0, MinAvailable: 0, NodesFitDelta: volcano.sh/volcano/pkg/scheduler/api.NodeResourceMap [], JobFitErrors: "", NodesFitErrors: map[volcano.sh/volcano/pkg/scheduler/api.TaskID]*volcano.sh/volcano/pkg/scheduler/api.FitErrors [], TaskStatusIndex: map[volcano.sh/volcano/pkg/scheduler/api.TaskStatus]volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Tasks: volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Allocated: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962e80), TotalRequest: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962ea0), CreationTimestamp: (*"k8s.io/apimachinery/pkg/apis/meta/v1.Time")(0x4000666088), PodGroup: *volcano.sh/volcano/pkg/scheduler/api.PodGroup nil,}, ] (dlv) p sc.Jobs.Tasks Command failed: sc.Jobs (type map[volcano.sh/volcano/pkg/scheduler/api.JobID]*volcano.sh/volcano/pkg/scheduler/api.JobInfo) is not a struct (dlv) p sc.Jobs.Tasks[0] Command failed: sc.Jobs (type map[volcano.sh/volcano/pkg/scheduler/api.JobID]*volcano.sh/volcano/pkg/scheduler/api.JobInfo) is not a struct (dlv) p sc.Jobs["default/job-1"].Tasks volcano.sh/volcano/pkg/scheduler/api.tasksMap [ "67387705-d4a1-4af6-ada7-85c4a40aaa27": *{ UID: "67387705-d4a1-4af6-ada7-85c4a40aaa27", Job: "default/job-1", Name: "job-1-nginx-0", Namespace: "default", Resreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962e00), InitResreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962e40), NodeName: "bogon", Status: Succeeded (128), Priority: 0, VolumeReady: false, Pod: *(*"k8s.io/api/core/v1.Pod")(0x4000b066c0),}, ] (dlv)
(dlv) c > volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:154 (hits goroutine(289):1 total:1) (PC: 0xe9a620) Warning: debugging optimized function (dlv) p tasks (unreadable read out of bounds) (dlv) args uid = "default/test-job" tasks = (unreadable read out of bounds) ~r2 = (unreadable empty OP stack) (dlv) s > volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:159 (PC: 0xe9a62c) Warning: debugging optimized function (dlv) p tasks (unreadable read out of bounds) (dlv) n > volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:163 (PC: 0xe9a668) Warning: debugging optimized function (dlv) n > volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:155 (PC: 0xe9a674) Warning: debugging optimized function (dlv) p tasks (unreadable read out of bounds) (dlv) c
再创建一个job
root@ubuntu:~# kubectl apply -f job2.yaml job.batch.volcano.sh/test-job-v2 created root@ubuntu:~# cat job2.yaml apiVersion: batch.volcano.sh/v1alpha1 kind: Job metadata: name: test-job-v2 spec: minAvailable: 1 schedulerName: volcano policies: - event: PodEvicted action: RestartJob plugins: ssh: [] env: [] svc: [] maxRetry: 5 queue: default tasks: - replicas: 1 name: "default-nginx-v2" template: metadata: name: web spec: containers: - image: nginx imagePullPolicy: IfNotPresent name: nginx resources: requests: cpu: "1" restartPolicy: OnFailure root@ubuntu:~#
(dlv) c > volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 (hits goroutine(289):2 total:2) (PC: 0xeab240) Warning: debugging optimized function (dlv) bt 0 0x0000000000eab240 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1 at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 1 0x0000000000eaeb68 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1-fm at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 2 0x0000000000c9ccc8 in k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:218 3 0x0000000000c9aed8 in k8s.io/client-go/tools/cache.(*processorListener).run.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:744 4 0x0000000000339ae4 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 5 0x0000000000338b54 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 6 0x0000000000338ac8 in k8s.io/apimachinery/pkg/util/wait.JitterUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 7 0x0000000000c96220 in k8s.io/apimachinery/pkg/util/wait.Until at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 8 0x0000000000c96220 in k8s.io/client-go/tools/cache.(*processorListener).run at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:738 9 0x0000000000c9c8c8 in k8s.io/client-go/tools/cache.(*processorListener).run-fm at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:732 10 0x0000000000339a44 in k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 11 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) p sc *volcano.sh/volcano/pkg/scheduler/cache.SchedulerCache { Mutex: sync.Mutex {state: 0, sema: 0}, kubeClient: *k8s.io/client-go/kubernetes.Clientset { DiscoveryClient: *(*"k8s.io/client-go/discovery.DiscoveryClient")(0x40005761c0), admissionregistrationV1: *(*"k8s.io/client-go/kubernetes/typed/admissionregistration/v1.AdmissionregistrationV1Client")(0x400050f380), admissionregistrationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1.AdmissionregistrationV1beta1Client")(0x400050f3e0), appsV1: *(*"k8s.io/client-go/kubernetes/typed/apps/v1.AppsV1Client")(0x400050f440), appsV1beta1: *(*"k8s.io/client-go/kubernetes/typed/apps/v1beta1.AppsV1beta1Client")(0x400050f4a0), appsV1beta2: *(*"k8s.io/client-go/kubernetes/typed/apps/v1beta2.AppsV1beta2Client")(0x400050f500), auditregistrationV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1.AuditregistrationV1alpha1Client")(0x400050f560), authenticationV1: *(*"k8s.io/client-go/kubernetes/typed/authentication/v1.AuthenticationV1Client")(0x400050f5c0), authenticationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/authentication/v1beta1.AuthenticationV1beta1Client")(0x400050f620), authorizationV1: *(*"k8s.io/client-go/kubernetes/typed/authorization/v1.AuthorizationV1Client")(0x400050f680), authorizationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/authorization/v1beta1.AuthorizationV1beta1Client")(0x400050f6e0), autoscalingV1: *(*"k8s.io/client-go/kubernetes/typed/autoscaling/v1.AutoscalingV1Client")(0x400050f740), autoscalingV2beta1: *(*"k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1.AutoscalingV2beta1Client")(0x400050f7a0), autoscalingV2beta2: *(*"k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2.AutoscalingV2beta2Client")(0x400050f800), batchV1: *(*"k8s.io/client-go/kubernetes/typed/batch/v1.BatchV1Client")(0x400050f860), batchV1beta1: *(*"k8s.io/client-go/kubernetes/typed/batch/v1beta1.BatchV1beta1Client")(0x400050f8c0), batchV2alpha1: *(*"k8s.io/client-go/kubernetes/typed/batch/v2alpha1.BatchV2alpha1Client")(0x400050f920), certificatesV1beta1: *(*"k8s.io/client-go/kubernetes/typed/certificates/v1beta1.CertificatesV1beta1Client")(0x400050f980), coordinationV1beta1: *(*"k8s.io/client-go/kubernetes/typed/coordination/v1beta1.CoordinationV1beta1Client")(0x400050f9e0), coordinationV1: *(*"k8s.io/client-go/kubernetes/typed/coordination/v1.CoordinationV1Client")(0x400050fa40), coreV1: *(*"k8s.io/client-go/kubernetes/typed/core/v1.CoreV1Client")(0x400050faa0), discoveryV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/discovery/v1alpha1.DiscoveryV1alpha1Client")(0x400050fb00), discoveryV1beta1: *(*"k8s.io/client-go/kubernetes/typed/discovery/v1beta1.DiscoveryV1beta1Client")(0x400050fb60), eventsV1beta1: *(*"k8s.io/client-go/kubernetes/typed/events/v1beta1.EventsV1beta1Client")(0x400050fbc0), extensionsV1beta1: *(*"k8s.io/client-go/kubernetes/typed/extensions/v1beta1.ExtensionsV1beta1Client")(0x400050fc20), flowcontrolV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1.FlowcontrolV1alpha1Client")(0x400050fc80), networkingV1: *(*"k8s.io/client-go/kubernetes/typed/networking/v1.NetworkingV1Client")(0x400050fce0), networkingV1beta1: *(*"k8s.io/client-go/kubernetes/typed/networking/v1beta1.NetworkingV1beta1Client")(0x400050fd40), nodeV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/node/v1alpha1.NodeV1alpha1Client")(0x400050fda0), nodeV1beta1: *(*"k8s.io/client-go/kubernetes/typed/node/v1beta1.NodeV1beta1Client")(0x400050fe00), policyV1beta1: *(*"k8s.io/client-go/kubernetes/typed/policy/v1beta1.PolicyV1beta1Client")(0x400050fe60), rbacV1: *(*"k8s.io/client-go/kubernetes/typed/rbac/v1.RbacV1Client")(0x400050fec0), rbacV1beta1: *(*"k8s.io/client-go/kubernetes/typed/rbac/v1beta1.RbacV1beta1Client")(0x400050ff20), rbacV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/rbac/v1alpha1.RbacV1alpha1Client")(0x400050ff80), schedulingV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1.SchedulingV1alpha1Client")(0x400050ffe0), schedulingV1beta1: *(*"k8s.io/client-go/kubernetes/typed/scheduling/v1beta1.SchedulingV1beta1Client")(0x400056e040), schedulingV1: *(*"k8s.io/client-go/kubernetes/typed/scheduling/v1.SchedulingV1Client")(0x400056e0a0), settingsV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/settings/v1alpha1.SettingsV1alpha1Client")(0x400056e100), storageV1beta1: *(*"k8s.io/client-go/kubernetes/typed/storage/v1beta1.StorageV1beta1Client")(0x400056e160), storageV1: *(*"k8s.io/client-go/kubernetes/typed/storage/v1.StorageV1Client")(0x400056e1c0), storageV1alpha1: *(*"k8s.io/client-go/kubernetes/typed/storage/v1alpha1.StorageV1alpha1Client")(0x400056e220),}, vcClient: *volcano.sh/volcano/pkg/client/clientset/versioned.Clientset { DiscoveryClient: *(*"k8s.io/client-go/discovery.DiscoveryClient")(0x4000576480), batchV1alpha1: *(*"volcano.sh/volcano/pkg/client/clientset/versioned/typed/batch/v1alpha1.BatchV1alpha1Client")(0x400056e330), busV1alpha1: *(*"volcano.sh/volcano/pkg/client/clientset/versioned/typed/bus/v1alpha1.BusV1alpha1Client")(0x400056e390), schedulingV1beta1: *(*"volcano.sh/volcano/pkg/client/clientset/versioned/typed/scheduling/v1beta1.SchedulingV1beta1Client")(0x400056e3f0),}, defaultQueue: "default", schedulerName: "volcano", podInformer: k8s.io/client-go/informers/core/v1.PodInformer(*k8s.io/client-go/informers/core/v1.podInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, nodeInformer: k8s.io/client-go/informers/core/v1.NodeInformer(*k8s.io/client-go/informers/core/v1.nodeInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, podGroupInformerV1beta1: volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.PodGroupInformer(*volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.podGroupInformer) *{ factory: volcano.sh/volcano/pkg/client/informers/externalversions/internalinterfaces.SharedInformerFactory(*volcano.sh/volcano/pkg/client/informers/externalversions.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, queueInformerV1beta1: volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.QueueInformer(*volcano.sh/volcano/pkg/client/informers/externalversions/scheduling/v1beta1.queueInformer) *{ factory: volcano.sh/volcano/pkg/client/informers/externalversions/internalinterfaces.SharedInformerFactory(*volcano.sh/volcano/pkg/client/informers/externalversions.sharedInformerFactory) ..., tweakListOptions: nil,}, pvInformer: k8s.io/client-go/informers/core/v1.PersistentVolumeInformer(*k8s.io/client-go/informers/core/v1.persistentVolumeInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, pvcInformer: k8s.io/client-go/informers/core/v1.PersistentVolumeClaimInformer(*k8s.io/client-go/informers/core/v1.persistentVolumeClaimInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, scInformer: k8s.io/client-go/informers/storage/v1.StorageClassInformer(*k8s.io/client-go/informers/storage/v1.storageClassInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, pcInformer: k8s.io/client-go/informers/scheduling/v1beta1.PriorityClassInformer(*k8s.io/client-go/informers/scheduling/v1beta1.priorityClassInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil,}, quotaInformer: k8s.io/client-go/informers/core/v1.ResourceQuotaInformer(*k8s.io/client-go/informers/core/v1.resourceQuotaInformer) *{ factory: k8s.io/client-go/informers/internalinterfaces.SharedInformerFactory(*k8s.io/client-go/informers.sharedInformerFactory) ..., tweakListOptions: nil, namespace: "",}, Binder: volcano.sh/volcano/pkg/scheduler/cache.Binder(*volcano.sh/volcano/pkg/scheduler/cache.defaultBinder) *{ kubeclient: *(*"k8s.io/client-go/kubernetes.Clientset")(0x4000548160),}, Evictor: volcano.sh/volcano/pkg/scheduler/cache.Evictor(*volcano.sh/volcano/pkg/scheduler/cache.defaultEvictor) *{ kubeclient: *(*"k8s.io/client-go/kubernetes.Clientset")(0x4000548160), recorder: k8s.io/client-go/tools/record.EventRecorder(*k8s.io/client-go/tools/record.recorderImpl) ...,}, StatusUpdater: volcano.sh/volcano/pkg/scheduler/cache.StatusUpdater(*volcano.sh/volcano/pkg/scheduler/cache.defaultStatusUpdater) *{ kubeclient: *(*"k8s.io/client-go/kubernetes.Clientset")(0x4000548160), vcclient: *(*"volcano.sh/volcano/pkg/client/clientset/versioned.Clientset")(0x4000576200),}, VolumeBinder: volcano.sh/volcano/pkg/scheduler/cache.VolumeBinder(*volcano.sh/volcano/pkg/scheduler/cache.defaultVolumeBinder) *{ volumeBinder: k8s.io/kubernetes/pkg/controller/volume/scheduling.SchedulerVolumeBinder(*k8s.io/kubernetes/pkg/controller/volume/scheduling.volumeBinder) ...,}, Recorder: k8s.io/client-go/tools/record.EventRecorder(*k8s.io/client-go/tools/record.recorderImpl) *{ scheme: *(*"k8s.io/apimachinery/pkg/runtime.Scheme")(0x4000509110), source: (*"k8s.io/api/core/v1.EventSource")(0x400013a8c8), Broadcaster: *(*"k8s.io/apimachinery/pkg/watch.Broadcaster")(0x400013a840), clock: k8s.io/apimachinery/pkg/util/clock.Clock(k8s.io/apimachinery/pkg/util/clock.RealClock) *(*"k8s.io/apimachinery/pkg/util/clock.Clock")(0x400013a8f0),}, Jobs: map[volcano.sh/volcano/pkg/scheduler/api.JobID]*volcano.sh/volcano/pkg/scheduler/api.JobInfo [ "default/test-job": *(*"volcano.sh/volcano/pkg/scheduler/api.JobInfo")(0x40003c80b0), "default/job-1": *(*"volcano.sh/volcano/pkg/scheduler/api.JobInfo")(0x4000666000), ], 没有新的job Nodes: map[string]*volcano.sh/volcano/pkg/scheduler/api.NodeInfo [ "bogon": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654880), "cloud": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654900), "centos7": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654980), "ubuntu": *(*"volcano.sh/volcano/pkg/scheduler/api.NodeInfo")(0x4000654a00), ], Queues: map[volcano.sh/volcano/pkg/scheduler/api.QueueID]*volcano.sh/volcano/pkg/scheduler/api.QueueInfo [ "default": *(*"volcano.sh/volcano/pkg/scheduler/api.QueueInfo")(0x40003636b0), "test": *(*"volcano.sh/volcano/pkg/scheduler/api.QueueInfo")(0x4000721680), ], PriorityClasses: map[string]*k8s.io/api/scheduling/v1beta1.PriorityClass [ "system-node-critical": *(*"k8s.io/api/scheduling/v1beta1.PriorityClass")(0x4000324a00), "system-cluster-critical": *(*"k8s.io/api/scheduling/v1beta1.PriorityClass")(0x4000324b38), ], defaultPriorityClass: *k8s.io/api/scheduling/v1beta1.PriorityClass nil, defaultPriority: 0, NamespaceCollection: map[string]*volcano.sh/volcano/pkg/scheduler/api.NamespaceCollection [], errTasks: k8s.io/client-go/util/workqueue.RateLimitingInterface(*k8s.io/client-go/util/workqueue.rateLimitingType) *{ DelayingInterface: k8s.io/client-go/util/workqueue.DelayingInterface(*k8s.io/client-go/util/workqueue.delayingType) ..., rateLimiter: k8s.io/client-go/util/workqueue.RateLimiter(*k8s.io/client-go/util/workqueue.MaxOfRateLimiter) ...,}, deletedJobs: k8s.io/client-go/util/workqueue.RateLimitingInterface(*k8s.io/client-go/util/workqueue.rateLimitingType) *{ DelayingInterface: k8s.io/client-go/util/workqueue.DelayingInterface(*k8s.io/client-go/util/workqueue.delayingType) ..., rateLimiter: k8s.io/client-go/util/workqueue.RateLimiter(*k8s.io/client-go/util/workqueue.MaxOfRateLimiter) ...,},}
(dlv) p sc.Jobs["default/test-job"].Tasks volcano.sh/volcano/pkg/scheduler/api.tasksMap [ "5d5b8e8d-3d05-4a9b-ab70-8a883d28f182": *{ UID: "5d5b8e8d-3d05-4a9b-ab70-8a883d28f182", Job: "default/test-job", Name: "test-job-default-nginx-3", Namespace: "default", Resreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000963f40), InitResreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000963f80), NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *(*"k8s.io/api/core/v1.Pod")(0x4000588c00),}, "7c37df2f-e1c0-479a-a5f9-900a2238878c": *{ UID: "7c37df2f-e1c0-479a-a5f9-900a2238878c", Job: "default/test-job", Name: "test-job-default-nginx-5", Namespace: "default", Resreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x40008d3d40), InitResreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x40008d3ea0), NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *(*"k8s.io/api/core/v1.Pod")(0x400073b800),}, "862f8ea1-c40e-41db-b70d-554f330a8187": *{ UID: "862f8ea1-c40e-41db-b70d-554f330a8187", Job: "default/test-job", Name: "test-job-default-nginx-4", Namespace: "default", Resreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000546660), InitResreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x40005466a0), NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *(*"k8s.io/api/core/v1.Pod")(0x4000589800),}, "6c19c994-6b58-4ac6-bd32-d5dc4a04b94f": *{ UID: "6c19c994-6b58-4ac6-bd32-d5dc4a04b94f", Job: "default/test-job", Name: "test-job-default-nginx-1", Namespace: "default", Resreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000215260), InitResreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x40002152c0), NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *(*"k8s.io/api/core/v1.Pod")(0x4000e24800),}, "1219f793-dd08-4569-b5b1-28ea58177725": *{ UID: "1219f793-dd08-4569-b5b1-28ea58177725", Job: "default/test-job", Name: "test-job-default-nginx-2", Namespace: "default", Resreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000214d00), InitResreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000214d40), NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *(*"k8s.io/api/core/v1.Pod")(0x4000e24000),}, "8e12d6dc-55c3-41cd-b4f9-f0f215faf09c": *{ UID: "8e12d6dc-55c3-41cd-b4f9-f0f215faf09c", Job: "default/test-job", Name: "test-job-default-nginx-0", Namespace: "default", Resreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x400050c600), InitResreq: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x400050c6c0), NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *(*"k8s.io/api/core/v1.Pod")(0x400075c000),}, ] (dlv)
(dlv) c > volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:154 (hits goroutine(289):2 total:2) (PC: 0xe9a620) Warning: debugging optimized function (dlv) p *task Command failed: could not find symbol value for task (dlv) s > volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:159 (PC: 0xe9a62c) Warning: debugging optimized function (dlv) args uid = "default/test-job-v2" 添加新的job tasks = (unreadable read out of bounds) ~r2 = (unreadable empty OP stack) (dlv) p *task Command failed: could not find symbol value for task
setPodGroup(
// Assumes that lock is already acquired. func (sc *SchedulerCache) setPodGroup(ss *schedulingapi.PodGroup) error { job := getJobID(ss) if _, found := sc.Jobs[job]; !found { sc.Jobs[job] = schedulingapi.NewJobInfo(job) } sc.Jobs[job].SetPodGroup(ss) // TODO(k82cn): set default queue in admission. if len(ss.Spec.Queue) == 0 { sc.Jobs[job].Queue = schedulingapi.QueueID(sc.defaultQueue) } return nil }
(dlv) c > volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).setPodGroup() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:356 (hits goroutine(289):1 total:1) (PC: 0xeaae10) Warning: debugging optimized function (dlv) bt 0 0x0000000000eaae10 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).setPodGroup at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:356 1 0x0000000000eab418 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1 at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:412 2 0x0000000000eaeb68 in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1-fm at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 3 0x0000000000c9ccc8 in k8s.io/client-go/tools/cache.(*ResourceEventHandlerFuncs).OnAdd at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/controller.go:218 4 0x0000000000c9aed8 in k8s.io/client-go/tools/cache.(*processorListener).run.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:744 5 0x0000000000339ae4 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 6 0x0000000000338b54 in k8s.io/apimachinery/pkg/util/wait.BackoffUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 7 0x0000000000338ac8 in k8s.io/apimachinery/pkg/util/wait.JitterUntil at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 8 0x0000000000c96220 in k8s.io/apimachinery/pkg/util/wait.Until at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 9 0x0000000000c96220 in k8s.io/client-go/tools/cache.(*processorListener).run at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:738 10 0x0000000000c9c8c8 in k8s.io/client-go/tools/cache.(*processorListener).run-fm at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/client-go/tools/cache/shared_informer.go:732 11 0x0000000000339a44 in k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1 at /usr/wulei/projects/src/volcano.sh/volcano/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:73 12 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) p ss *volcano.sh/volcano/pkg/scheduler/api.PodGroup { PodGroup: volcano.sh/volcano/pkg/apis/scheduling.PodGroup { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000980000), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000980020), Spec: (*"volcano.sh/volcano/pkg/apis/scheduling.PodGroupSpec")(0x4000980118), Status: (*"volcano.sh/volcano/pkg/apis/scheduling.PodGroupStatus")(0x4000980148),}, Version: "v1beta1",} (dlv) p ss.ObjectMeta k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta { Name: "test-job-v2", GenerateName: "", Namespace: "default", SelfLink: "/apis/scheduling.volcano.sh/v1beta1/namespaces/default/podgroups...+12 more", UID: "f3c08856-c74c-4233-ad7d-d3e11f2b7899", ResourceVersion: "3033546", Generation: 1, CreationTimestamp: k8s.io/apimachinery/pkg/apis/meta/v1.Time { Time: (*time.Time)(0x4000980088),}, DeletionTimestamp: *k8s.io/apimachinery/pkg/apis/meta/v1.Time nil, DeletionGracePeriodSeconds: *int64 nil, Labels: map[string]string nil, Annotations: map[string]string [ "kubectl.kubernetes.io/last-applied-configuration": "{"apiVersion":"batch.volcano.sh/v1alpha1","kind":"Job","metadata...+490 more", ], OwnerReferences: []k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.OwnerReference")(0x4000284280), ], Finalizers: []string len: 0, cap: 0, nil, ClusterName: "", ManagedFields: []k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry len: 1, cap: 1, [ (*"k8s.io/apimachinery/pkg/apis/meta/v1.ManagedFieldsEntry")(0x4000284230), ],} (dlv) p sc.Jobs map[volcano.sh/volcano/pkg/scheduler/api.JobID]*volcano.sh/volcano/pkg/scheduler/api.JobInfo [ "default/test-job": *{ UID: "default/test-job", Name: "test-job", Namespace: "default", Queue: "default", Priority: 0, MinAvailable: 3, NodesFitDelta: volcano.sh/volcano/pkg/scheduler/api.NodeResourceMap [], JobFitErrors: "", NodesFitErrors: map[volcano.sh/volcano/pkg/scheduler/api.TaskID]*volcano.sh/volcano/pkg/scheduler/api.FitErrors [], TaskStatusIndex: map[volcano.sh/volcano/pkg/scheduler/api.TaskStatus]volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Tasks: volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Allocated: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x40001ba060), TotalRequest: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x40001ba0e0), CreationTimestamp: (*"k8s.io/apimachinery/pkg/apis/meta/v1.Time")(0x40003c8138), PodGroup: *(*"volcano.sh/volcano/pkg/scheduler/api.PodGroup")(0x4000e6f1e0),}, "default/job-1": *{ UID: "default/job-1", Name: "", Namespace: "", Queue: "", Priority: 0, MinAvailable: 0, NodesFitDelta: volcano.sh/volcano/pkg/scheduler/api.NodeResourceMap [], JobFitErrors: "", NodesFitErrors: map[volcano.sh/volcano/pkg/scheduler/api.TaskID]*volcano.sh/volcano/pkg/scheduler/api.FitErrors [], TaskStatusIndex: map[volcano.sh/volcano/pkg/scheduler/api.TaskStatus]volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Tasks: volcano.sh/volcano/pkg/scheduler/api.tasksMap [...], Allocated: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962e80), TotalRequest: *(*"volcano.sh/volcano/pkg/scheduler/api.Resource")(0x4000962ea0), CreationTimestamp: (*"k8s.io/apimachinery/pkg/apis/meta/v1.Time")(0x4000666088), PodGroup: *volcano.sh/volcano/pkg/scheduler/api.PodGroup nil,}, ] (dlv)
root@ubuntu:~# kubectl get pods NAME READY STATUS RESTARTS AGE example-foo-54dc4db9fc-lqz9j 1/1 Running 0 4d20h job-1-nginx-0 0/1 Completed 0 7d4h test-job-default-nginx-0 1/1 Running 0 20m test-job-default-nginx-1 1/1 Running 0 20m test-job-default-nginx-2 1/1 Running 0 20m test-job-default-nginx-3 1/1 Running 0 20m test-job-default-nginx-4 1/1 Running 0 20m test-job-default-nginx-5 1/1 Running 0 20m test-job-v2-default-nginx-v2-0 1/1 Running 0 7m18s root@ubuntu:~#
job clone
(dlv) bt 0 0x0000000000e9a960 in volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 1 0x0000000000e9b120 in volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).Clone at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:286 2 0x0000000000eae46c in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).Snapshot.func1 at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/cache.go:708 3 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) p task Command failed: could not find symbol value for task (dlv) Command failed: could not find symbol value for task (dlv) p ji *volcano.sh/volcano/pkg/scheduler/api.JobInfo { UID: "default/test-job", Name: "test-job", Namespace: "default", Queue: "default", Priority: 0, MinAvailable: 3, NodesFitDelta: volcano.sh/volcano/pkg/scheduler/api.NodeResourceMap [], JobFitErrors: "", NodesFitErrors: map[volcano.sh/volcano/pkg/scheduler/api.TaskID]*volcano.sh/volcano/pkg/scheduler/api.FitErrors [], TaskStatusIndex: map[volcano.sh/volcano/pkg/scheduler/api.TaskStatus]volcano.sh/volcano/pkg/scheduler/api.tasksMap [], Tasks: volcano.sh/volcano/pkg/scheduler/api.tasksMap [], Allocated: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 0, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, TotalRequest: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 0, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, CreationTimestamp: k8s.io/apimachinery/pkg/apis/meta/v1.Time { Time: (*time.Time)(0x40003f6088),}, PodGroup: *volcano.sh/volcano/pkg/scheduler/api.PodGroup { PodGroup: (*"volcano.sh/volcano/pkg/apis/scheduling.PodGroup")(0x4000e6f1e0), Version: "v1beta1",},} (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "5d5b8e8d-3d05-4a9b-ab70-8a883d28f182", Job: "default/test-job", Name: "test-job-default-nginx-3", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000588c00), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000588c20), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000588d18), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000588ef0),},} (dlv) bt 0 0x0000000000e9a960 in volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 1 0x0000000000e9b120 in volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).Clone at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:286 2 0x0000000000eae46c in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).Snapshot.func1 at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/cache.go:708 3 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) bp Breakpoint runtime-fatal-throw (enabled) at 0x424b0 for runtime.fatalthrow() /usr/local/go/src/runtime/panic.go:1162 (0) Breakpoint unrecovered-panic (enabled) at 0x42510 for runtime.fatalpanic() /usr/local/go/src/runtime/panic.go:1189 (0) print runtime.curg._panic.arg Breakpoint 1 (enabled) at 0xeab240 for volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 (2) Breakpoint 2 (enabled) at 0xe9a620 for volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:154 (2) Breakpoint 3 (enabled) at 0xe9a960 for volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (1) (dlv) clear 1 Breakpoint 1 (enabled) cleared at 0xeab240 for volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).AddPodGroupV1beta1() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/event_handlers.go:393 (dlv) clear 2 Breakpoint 2 (enabled) cleared at 0xe9a620 for volcano.sh/volcano/pkg/scheduler/api.NewJobInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:154 (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35749):2 total:2) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "7c37df2f-e1c0-479a-a5f9-900a2238878c", Job: "default/test-job", Name: "test-job-default-nginx-5", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x400073b800), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x400073b820), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x400073b918), Status: (*"k8s.io/api/core/v1.PodStatus")(0x400073baf0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35749):3 total:3) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "862f8ea1-c40e-41db-b70d-554f330a8187", Job: "default/test-job", Name: "test-job-default-nginx-4", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000589800), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000589820), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000589918), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000589af0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35749):4 total:4) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "6c19c994-6b58-4ac6-bd32-d5dc4a04b94f", Job: "default/test-job", Name: "test-job-default-nginx-1", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000e24800), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000e24820), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000e24918), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000e24af0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35749):5 total:5) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "1219f793-dd08-4569-b5b1-28ea58177725", Job: "default/test-job", Name: "test-job-default-nginx-2", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000e24000), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000e24020), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000e24118), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000e242f0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35749):6 total:6) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "8e12d6dc-55c3-41cd-b4f9-f0f215faf09c", Job: "default/test-job", Name: "test-job-default-nginx-0", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x400075c000), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x400075c020), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x400075c118), Status: (*"k8s.io/api/core/v1.PodStatus")(0x400075c2f0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35789):1 total:7) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "8e12d6dc-55c3-41cd-b4f9-f0f215faf09c", Job: "default/test-job", Name: "test-job-default-nginx-0", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x400075c000), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x400075c020), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x400075c118), Status: (*"k8s.io/api/core/v1.PodStatus")(0x400075c2f0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35789):2 total:8) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "5d5b8e8d-3d05-4a9b-ab70-8a883d28f182", Job: "default/test-job", Name: "test-job-default-nginx-3", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000588c00), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000588c20), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000588d18), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000588ef0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35789):3 total:9) (PC: 0xe9a960) Warning: debugging optimized function (dlv) bt 0 0x0000000000e9a960 in volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 1 0x0000000000e9b120 in volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).Clone at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:286 2 0x0000000000eae46c in volcano.sh/volcano/pkg/scheduler/cache.(*SchedulerCache).Snapshot.func1 at /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/cache/cache.go:708 3 0x0000000000073ff4 in runtime.goexit at /usr/local/go/src/runtime/asm_arm64.s:1148 (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "7c37df2f-e1c0-479a-a5f9-900a2238878c", Job: "default/test-job", Name: "test-job-default-nginx-5", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x400073b800), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x400073b820), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x400073b918), Status: (*"k8s.io/api/core/v1.PodStatus")(0x400073baf0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35789):4 total:10) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "862f8ea1-c40e-41db-b70d-554f330a8187", Job: "default/test-job", Name: "test-job-default-nginx-4", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000589800), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000589820), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000589918), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000589af0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35789):5 total:11) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "6c19c994-6b58-4ac6-bd32-d5dc4a04b94f", Job: "default/test-job", Name: "test-job-default-nginx-1", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000e24800), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000e24820), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000e24918), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000e24af0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35789):6 total:12) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "1219f793-dd08-4569-b5b1-28ea58177725", Job: "default/test-job", Name: "test-job-default-nginx-2", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000e24000), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000e24020), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000e24118), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000e242f0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(174):1 total:13) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "4f464538-1316-4e81-8418-818764249fcf", Job: "default/test-job-v2", Name: "test-job-v2-default-nginx-v2-0", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "", Status: Pending (1), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000a15400), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000a15420), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000a15518), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000a156f0),},} (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35834):1 total:15) (PC: 0xe9a960) Warning: debugging optimized function > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35833):1 total:15) (PC: 0xe9a960) Warning: debugging optimized function (dlv) c > volcano.sh/volcano/pkg/scheduler/api.(*JobInfo).AddTaskInfo() /usr/wulei/projects/src/volcano.sh/volcano/pkg/scheduler/api/job_info.go:200 (hits goroutine(35833):2 total:16) (PC: 0xe9a960) Warning: debugging optimized function (dlv) p ti *volcano.sh/volcano/pkg/scheduler/api.TaskInfo { UID: "862f8ea1-c40e-41db-b70d-554f330a8187", Job: "default/test-job", Name: "test-job-default-nginx-4", Namespace: "default", Resreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, InitResreq: *volcano.sh/volcano/pkg/scheduler/api.Resource { MilliCPU: 1000, Memory: 0, ScalarResources: map[k8s.io/api/core/v1.ResourceName]float64 nil, MaxTaskNum: 0,}, NodeName: "bogon", Status: Running (32), Priority: 0, VolumeReady: false, Pod: *k8s.io/api/core/v1.Pod { TypeMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta")(0x4000589800), ObjectMeta: (*"k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta")(0x4000589820), Spec: (*"k8s.io/api/core/v1.PodSpec")(0x4000589918), Status: (*"k8s.io/api/core/v1.PodStatus")(0x4000589af0),},} (dlv) quit Would you like to kill the process? [Y/n] n You have mail in /var/spool/mail/root [root@bogon ~]#
OpenSession
用来启动 Session 作为调度的上下文，并且通过 tiers 的配置信息创建不同的 plugins，然后再通过 OnSessionOpen 函数，注册不同的辅助函数位点，如 jobOrderFns，queueOrderFns 等。其中 openSession 函数用于生成一个 session 实例，然后对 schedulerCache 进行一次 snapshot（把 cache 中的 jobs，queues 等信息使用snapshot 缓存起来），然后传给 session。
// pkg/scheduler/framework/framework.go func OpenSession(cache cache.Cache, tiers []conf.Tier, configurations []conf.Configuration) *Session { ssn := openSession(cache) ssn.Tiers = tiers ssn.Configurations = configurations for _, tier := range tiers { for _, plugin := range tier.Plugins { if pb, found := GetPluginBuilder(plugin.Name); !found { klog.Errorf("Failed to get plugin %s.", plugin.Name) } else { plugin := pb(plugin.Arguments) ssn.plugins[plugin.Name()] = plugin } } } for _, plugin := range ssn.plugins { onSessionOpenStart := time.Now() plugin.OnSessionOpen(ssn) metrics.UpdatePluginDuration(plugin.Name(), metrics.OnSessionOpen, metrics.Duration(onSessionOpenStart)) } return ssn } // pkg/scheduler/framework/session.go func openSession(cache cache.Cache) *Session { ssn := &Session{ UID: uuid.NewUUID(), kubeClient: cache.Client(), cache: cache, podGroupStatus: map[api.JobID]scheduling.PodGroupStatus{}, Jobs: map[api.JobID]*api.JobInfo{}, Nodes: map[string]*api.NodeInfo{}, Queues: map[api.QueueID]*api.QueueInfo{}, ... // 操作函数位点 } snapshot := cache.Snapshot() ssn.Jobs = snapshot.Jobs for _, job := range ssn.Jobs { // only conditions will be updated periodically if job.PodGroup != nil && job.PodGroup.Status.Conditions != nil { ssn.podGroupStatus[job.UID] = job.PodGroup.Status } if vjr := ssn.JobValid(job); vjr != nil { if !vjr.Pass { ... // 如果这个 job 无效 } delete(ssn.Jobs, job.UID) } } ssn.Nodes = snapshot.Nodes ssn.Queues = snapshot.Queues ssn.NamespaceInfo = snapshot.NamespaceInfo return ssn }
snapshot
// Snapshot returns the complete snapshot of the cluster from cache func (sc *SchedulerCache) Snapshot() *kbapi.ClusterInfo { sc.Mutex.Lock() defer sc.Mutex.Unlock() snapshot := &kbapi.ClusterInfo{ Nodes: make(map[string]*kbapi.NodeInfo), Jobs: make(map[kbapi.JobID]*kbapi.JobInfo), Queues: make(map[kbapi.QueueID]*kbapi.QueueInfo), } for _, value := range sc.Nodes { if !value.Ready() { continue } snapshot.Nodes[value.Name] = value.Clone() } for _, value := range sc.Queues { snapshot.Queues[value.UID] = value.Clone() } for _, value := range sc.Jobs { // If no scheduling spec, does not handle it. if value.PodGroup == nil && value.PDB == nil { glog.V(4).Infof("The scheduling spec of Job <%v:%s/%s> is nil, ignore it.", value.UID, value.Namespace, value.Name) continue } if _, found := snapshot.Queues[value.Queue]; !found { glog.V(3).Infof("The Queue <%v> of Job <%v/%v> does not exist, ignore it.", value.Queue, value.Namespace, value.Name) continue } if value.PodGroup != nil { value.Priority = sc.defaultPriority priName := value.PodGroup.Spec.PriorityClassName if priorityClass, found := sc.PriorityClasses[priName]; found { value.Priority = priorityClass.Value } glog.V(4).Infof("The priority of job <%s/%s> is <%s/%d>", value.Namespace, value.Name, priName, value.Priority) } snapshot.Jobs[value.UID] = value.Clone() } glog.V(3).Infof("There are <%d> Jobs, <%d> Queues and <%d> Nodes in total for scheduling.", len(snapshot.Jobs), len(snapshot.Queues), len(snapshot.Nodes)) return snapshot }