新增kpl-launcher服务

kpl_deploy_yaml/5-kpl-launcher/certs/server.crt

+-----BEGIN CERTIFICATE-----
+MIICKTCCAbCgAwIBAgIJAOEzff/TB45/MAoGCCqGSM49BAMCMFMxCzAJBgNVBAYT
+AkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRn
+aXRzIFB0eSBMdGQxDDAKBgNVBAMMA0tQTDAeFw0yMDA2MDMwMzA2MDRaFw0zMDA2
+MDEwMzA2MDRaMFMxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDDAKBgNVBAMMA0tQTDB2
+MBAGByqGSM49AgEGBSuBBAAiA2IABDUlvu7GTOB4kYiqBuRqiLU3chqccZhMFL16
+olmMX31M8EWA0VXj5JeMo4js7NcuBRfFp2JIdhqOroodz+Bu64nmhjbr7Qkglk14
+XguoUrwycTAlba2JdBpXRXwY5uP7eqNQME4wHQYDVR0OBBYEFPQ81JjaE8UG4FyX
+Hjo09H9dRkcEMB8GA1UdIwQYMBaAFPQ81JjaE8UG4FyXHjo09H9dRkcEMAwGA1Ud
+EwQFMAMBAf8wCgYIKoZIzj0EAwIDZwAwZAIwSCzsAdwv5fJOlAMI6W+0s5whygR3
+VQEq88EffPmjQ8Cn6rqWFzev4Cd5W18Qput9AjAjoBh5WdlK1N0sIZpRLaCYK7El
+2vab3X1CbV8MkwGJU7Vnjav+w185kSNpbpF6idw=
+-----END CERTIFICATE-----

kpl_deploy_yaml/5-kpl-launcher/certs/server.key

+-----BEGIN EC PARAMETERS-----
+BgUrgQQAIg==
+-----END EC PARAMETERS-----
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDCc8hpwAUrmEZUnFeD4Fi/OnMT2fAXVtJ50FIR/HCWMD/pPDV1uKLZI
+Hm6h6fRQX82gBwYFK4EEACKhZANiAAQ1Jb7uxkzgeJGIqgbkaoi1N3IanHGYTBS9
+eqJZjF99TPBFgNFV4+SXjKOI7OzXLgUXxadiSHYajq6KHc/gbuuJ5oY26+0JIJZN
+eF4LqFK8MnEwJW2tiXQaV0V8GObj+3o=
+-----END EC PRIVATE KEY-----

kpl_deploy_yaml/5-kpl-launcher/kpl-launcher/deployment.yaml

+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: kpl-launcher
+  namespace: kpl
+  labels:
+    app: kpl-launcher
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kpl-launcher
+  template:
+    metadata:
+      labels:
+        app: kpl-launcher
+    spec:
+      serviceAccount: kpl-launcher
+      containers:
+        - name: launcher
+          image: IMAGE_NAME   #镜像仓库以及名字变量模板
+          command:
+            - /bin/bash
+            - -c
+            - kpl_launcher --incluster --private-key /etc/kpl/ssl/server.key --cert-chain /etc/kpl/ssl/server.crt --port 8000 2>&1
+          ports:
+            - containerPort: 8000
+              name: launcher-port
+          imagePullPolicy: "IfNotPresent"
+          resources:
+              requests:   #新增加request。降低资源调度要求
+                cpu: 1
+                memory: 100Mi
+              limits:
+                cpu: 8
+                memory: 100Mi
+          env:
+            - name: KPL_IMAGE_SECRET_NAME
+              value: kpl-regcred
+          volumeMounts:
+            - name: kpl-ssl
+              mountPath: /etc/kpl/ssl
+              readOnly: true
+      volumes:
+        - name: kpl-ssl
+          configMap:
+            name: kpl-ssl
+      imagePullSecrets:
+        - name: kpl-regcred
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: kpl-launcher
+  name: kpl-launcher-service
+  namespace: kpl
+spec:
+  ports:
+    - port: 8000
+      protocol: TCP
+      targetPort: 8000
+  # type: NodePort
+  selector:
+    app: kpl-launcher

kpl_deploy_yaml/5-kpl-launcher/kpl-launcher/rbac.yaml

+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kpl-launcher
+  namespace: kpl
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kpl-launcher
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch.volcano.sh"]
+    resources: ["jobs"]
+    verbs: ["get", "create", "list", "watch", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["pods", "pods/status"]
+    verbs: ["create", "get", "list", "watch", "update", "bind", "updateStatus", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: kpl-launcher
+subjects:
+- kind: ServiceAccount
+  name: kpl-launcher
+  namespace: kpl
+roleRef:
+  kind: ClusterRole
+  name: kpl-launcher
+  apiGroup: rbac.authorization.k8s.io

kpl_deploy_yaml/5-kpl-launcher/kpl-ssl-configmap-autodl.yaml

+apiVersion: v1
+data:
+  server.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIICKTCCAbCgAwIBAgIJAOEzff/TB45/MAoGCCqGSM49BAMCMFMxCzAJBgNVBAYT
+    AkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRn
+    aXRzIFB0eSBMdGQxDDAKBgNVBAMMA0tQTDAeFw0yMDA2MDMwMzA2MDRaFw0zMDA2
+    MDEwMzA2MDRaMFMxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+    HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDDAKBgNVBAMMA0tQTDB2
+    MBAGByqGSM49AgEGBSuBBAAiA2IABDUlvu7GTOB4kYiqBuRqiLU3chqccZhMFL16
+    olmMX31M8EWA0VXj5JeMo4js7NcuBRfFp2JIdhqOroodz+Bu64nmhjbr7Qkglk14
+    XguoUrwycTAlba2JdBpXRXwY5uP7eqNQME4wHQYDVR0OBBYEFPQ81JjaE8UG4FyX
+    Hjo09H9dRkcEMB8GA1UdIwQYMBaAFPQ81JjaE8UG4FyXHjo09H9dRkcEMAwGA1Ud
+    EwQFMAMBAf8wCgYIKoZIzj0EAwIDZwAwZAIwSCzsAdwv5fJOlAMI6W+0s5whygR3
+    VQEq88EffPmjQ8Cn6rqWFzev4Cd5W18Qput9AjAjoBh5WdlK1N0sIZpRLaCYK7El
+    2vab3X1CbV8MkwGJU7Vnjav+w185kSNpbpF6idw=
+    -----END CERTIFICATE-----
+  server.key: |
+    -----BEGIN EC PARAMETERS-----
+    BgUrgQQAIg==
+    -----END EC PARAMETERS-----
+    -----BEGIN EC PRIVATE KEY-----
+    MIGkAgEBBDCc8hpwAUrmEZUnFeD4Fi/OnMT2fAXVtJ50FIR/HCWMD/pPDV1uKLZI
+    Hm6h6fRQX82gBwYFK4EEACKhZANiAAQ1Jb7uxkzgeJGIqgbkaoi1N3IanHGYTBS9
+    eqJZjF99TPBFgNFV4+SXjKOI7OzXLgUXxadiSHYajq6KHc/gbuuJ5oY26+0JIJZN
+    eF4LqFK8MnEwJW2tiXQaV0V8GObj+3o=
+    -----END EC PRIVATE KEY-----
+kind: ConfigMap
+metadata:
+  name: kpl-ssl
+  namespace: autodl

kpl_deploy_yaml/5-kpl-launcher/kpl-ssl-configmap.yaml

+apiVersion: v1
+data:
+  server.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIICKTCCAbCgAwIBAgIJAOEzff/TB45/MAoGCCqGSM49BAMCMFMxCzAJBgNVBAYT
+    AkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRn
+    aXRzIFB0eSBMdGQxDDAKBgNVBAMMA0tQTDAeFw0yMDA2MDMwMzA2MDRaFw0zMDA2
+    MDEwMzA2MDRaMFMxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+    HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxDDAKBgNVBAMMA0tQTDB2
+    MBAGByqGSM49AgEGBSuBBAAiA2IABDUlvu7GTOB4kYiqBuRqiLU3chqccZhMFL16
+    olmMX31M8EWA0VXj5JeMo4js7NcuBRfFp2JIdhqOroodz+Bu64nmhjbr7Qkglk14
+    XguoUrwycTAlba2JdBpXRXwY5uP7eqNQME4wHQYDVR0OBBYEFPQ81JjaE8UG4FyX
+    Hjo09H9dRkcEMB8GA1UdIwQYMBaAFPQ81JjaE8UG4FyXHjo09H9dRkcEMAwGA1Ud
+    EwQFMAMBAf8wCgYIKoZIzj0EAwIDZwAwZAIwSCzsAdwv5fJOlAMI6W+0s5whygR3
+    VQEq88EffPmjQ8Cn6rqWFzev4Cd5W18Qput9AjAjoBh5WdlK1N0sIZpRLaCYK7El
+    2vab3X1CbV8MkwGJU7Vnjav+w185kSNpbpF6idw=
+    -----END CERTIFICATE-----
+  server.key: |
+    -----BEGIN EC PARAMETERS-----
+    BgUrgQQAIg==
+    -----END EC PARAMETERS-----
+    -----BEGIN EC PRIVATE KEY-----
+    MIGkAgEBBDCc8hpwAUrmEZUnFeD4Fi/OnMT2fAXVtJ50FIR/HCWMD/pPDV1uKLZI
+    Hm6h6fRQX82gBwYFK4EEACKhZANiAAQ1Jb7uxkzgeJGIqgbkaoi1N3IanHGYTBS9
+    eqJZjF99TPBFgNFV4+SXjKOI7OzXLgUXxadiSHYajq6KHc/gbuuJ5oY26+0JIJZN
+    eF4LqFK8MnEwJW2tiXQaV0V8GObj+3o=
+    -----END EC PRIVATE KEY-----
+kind: ConfigMap
+metadata:
+  creationTimestamp: "2020-06-03T03:06:22Z"
+  name: kpl-ssl
+  namespace: kpl
+  resourceVersion: "61559587"
+  selfLink: /api/v1/namespaces/kpl/configmaps/kpl-ssl
+  uid: 4c6174b1-3847-4c29-a7e7-fcd7b6e011e9

kpl_deploy_yaml/5-kpl-launcher/volcano/volcano-development.yaml

+# Source: volcano/templates/scheduler.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: volcano-scheduler-configmap
+  namespace: volcano-system
+data:
+  volcano-scheduler.conf: |
+    actions: "enqueue, allocate, backfill"
+    tiers:
+    - plugins:
+      - name: priority
+      - name: gang
+      - name: conformance
+    - plugins:
+      - name: drf
+      - name: predicates
+      - name: proportion
+      - name: nodeorder
+      - name: binpack
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: volcano-scheduler
+  namespace: volcano-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: volcano-scheduler
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "get", "list", "watch", "delete"]
+  - apiGroups: ["batch.volcano.sh"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "update", "delete"]
+  - apiGroups: ["batch.volcano.sh"]
+    resources: ["jobs/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["pods", "pods/status"]
+    verbs: ["create", "get", "list", "watch", "update", "bind", "updateStatus", "delete"]
+  - apiGroups: [""]
+    resources: ["pods/binding"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["resourcequotas"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["list", "watch"]
+  - apiGroups: ["scheduling.k8s.io"]
+    resources: ["priorityclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
+    resources: ["queues"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
+    resources: ["podgroups"]
+    verbs: ["list", "watch", "update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: volcano-scheduler-role
+subjects:
+  - kind: ServiceAccount
+    name: volcano-scheduler
+    namespace: volcano-system
+roleRef:
+  kind: ClusterRole
+  name: volcano-scheduler
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: volcano-scheduler
+  namespace: volcano-system
+  labels:
+    app: volcano-scheduler
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: volcano-scheduler
+  template:
+    metadata:
+      labels:
+        app: volcano-scheduler
+    spec:
+      serviceAccount: volcano-scheduler
+
+      containers:
+        - name: volcano-scheduler
+          image: hub.kce.ksyun.com/aivc/volcanosh/vc-scheduler:latest
+          args:
+            - --logtostderr
+            - --scheduler-conf=/volcano.scheduler/volcano-scheduler.conf
+            - -v=3
+            - 2>&1
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: scheduler-config
+              mountPath: /volcano.scheduler
+      volumes:
+        - name: scheduler-config
+          configMap:
+            name: volcano-scheduler-configmap
+
+---
+# Source: volcano/templates/admission.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: volcano-admission
+  namespace: volcano-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: volcano-admission
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["admissionregistration.k8s.io"]
+    resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"]
+    verbs: ["get", "list", "watch", "create", "update"]
+  # Rules below is used generate admission service secret
+  - apiGroups: ["certificates.k8s.io"]
+    resources: ["certificatesigningrequests"]
+    verbs: ["get", "list", "create", "delete"]
+  - apiGroups: ["certificates.k8s.io"]
+    resources: ["certificatesigningrequests/approval"]
+    verbs: ["create", "update"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "get", "patch"]
+  - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
+    resources: ["queues"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["get"]
+  - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
+    resources: ["podgroups"]
+    verbs: ["get", "list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: volcano-admission-role
+subjects:
+  - kind: ServiceAccount
+    name: volcano-admission
+    namespace: volcano-system
+roleRef:
+  kind: ClusterRole
+  name: volcano-admission
+  apiGroup: rbac.authorization.k8s.io
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: volcano-admission
+  name: volcano-admission
+  namespace: volcano-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: volcano-admission
+  template:
+    metadata:
+      labels:
+        app: volcano-admission
+    spec:
+      serviceAccount: volcano-admission
+
+      containers:
+        - args:
+            - --tls-cert-file=/admission.local.config/certificates/tls.crt
+            - --tls-private-key-file=/admission.local.config/certificates/tls.key
+            - --ca-cert-file=/admission.local.config/certificates/ca.crt
+            - --webhook-namespace=volcano-system
+            - --webhook-service-name=volcano-admission-service
+            - --logtostderr
+            - --port=8443
+            - -v=4
+            - 2>&1
+          image: hub.kce.ksyun.com/aivc/volcanosh/vc-webhook-manager:latest
+          imagePullPolicy: IfNotPresent
+          name: admission
+          volumeMounts:
+            - mountPath: /admission.local.config/certificates
+              name: admission-certs
+              readOnly: true
+      volumes:
+        - name: admission-certs
+          secret:
+            defaultMode: 420
+            secretName: volcano-admission-secret
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: volcano-admission
+  name: volcano-admission-service
+  namespace: volcano-system
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 8443
+  selector:
+    app: volcano-admission
+  sessionAffinity: None
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: volcano-admission-init
+  namespace: volcano-system
+  labels:
+    app: volcano-admission-init
+spec:
+  backoffLimit: 3
+  template:
+    spec:
+      serviceAccountName: volcano-admission
+      restartPolicy: Never
+      containers:
+        - name: main
+          image: hub.kce.ksyun.com/aivc/volcanosh/vc-webhook-manager:latest
+          imagePullPolicy: IfNotPresent
+          command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace",
+                    "volcano-system", "--secret", "volcano-admission-secret"]
+
+---
+# Source: volcano/templates/controllers.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: volcano-controllers
+  namespace: volcano-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: volcano-controllers
+rules:
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "get", "list", "watch", "delete"]
+  - apiGroups: ["batch.volcano.sh"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "update", "delete"]
+  - apiGroups: ["batch.volcano.sh"]
+    resources: ["jobs/status", "jobs/finalizers"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["bus.volcano.sh"]
+    resources: ["commands"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["create", "get", "list", "watch", "update", "bind", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "create"]
+  - apiGroups: [""]
+    resources: ["services"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch", "create", "delete", "update"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch", "create", "delete", "update"]
+  - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
+    resources: ["podgroups", "queues", "queues/status"]
+    verbs: ["get", "list", "watch", "create", "delete", "update"]
+  - apiGroups: ["scheduling.k8s.io"]
+    resources: ["priorityclasses"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["networkpolicies"]
+    verbs: ["get", "create"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: volcano-controllers-role
+subjects:
+  - kind: ServiceAccount
+    name: volcano-controllers
+    namespace: volcano-system
+roleRef:
+  kind: ClusterRole
+  name: volcano-controllers
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: volcano-controllers
+  namespace: volcano-system
+  labels:
+    app: volcano-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: volcano-controller
+  template:
+    metadata:
+      labels:
+        app: volcano-controller
+    spec:
+      serviceAccount: volcano-controllers
+
+      containers:
+          - name: volcano-controllers
+            image: hub.kce.ksyun.com/aivc/volcanosh/vc-controller-manager:latest
+            args:
+              - --logtostderr
+              - -v=4
+              - 2>&1
+            imagePullPolicy: "IfNotPresent"
+
+---
+# Source: volcano/templates/batch_v1alpha1_job.yaml
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: jobs.batch.volcano.sh
+spec:
+  group: batch.volcano.sh
+  names:
+    kind: Job
+    plural: jobs
+    shortNames:
+      - vcjob
+      - vj
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      type: object
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: Specification of the desired behavior of a cron job, including
+            the minAvailable
+          properties:
+            volumes:
+              description: The volumes for Job
+              items:
+                properties:
+                  volumeClaim:
+                    description: VolumeClaim defines the PVC used by the VolumeMount.
+                    type: object
+                  mountPath:
+                    description: Path within the container at which the volume should be mounted.
+                      Must not contain ':'.
+                    type: string
+                  volumeClaimName:
+                    description: The name of the volume claim.
+                    type: string
+                type: object
+                required:
+                  - mountPath
+              type: array
+            minAvailable:
+              description: The minimal available pods to run for this Job
+              format: int32
+              type: integer
+            policies:
+              description: Specifies the default lifecycle of tasks
+              items:
+                properties:
+                  action:
+                    description: The action that will be taken to the PodGroup according
+                      to Event. One of "Restart", "None". Default to None.
+                    type: string
+                  event:
+                    description: The Event recorded by scheduler; the controller takes
+                      actions according to this Event.
+                    type: string
+                  events:
+                    description: The Events recorded by scheduler; the controller takes
+                      actions according to this Events.
+                    type: array
+                    items:
+                      type: string
+                  timeout:
+                    description: Timeout is the grace period for controller to take
+                      actions. Default to nil (take action immediately).
+                    type: object
+                type: object
+              type: array
+            schedulerName:
+              description: SchedulerName is the default value of `tasks.template.spec.schedulerName`.
+              type: string
+            plugins:
+              description: Enabled task plugins when creating job.
+              type: object
+            tasks:
+              description: Tasks specifies the task specification of Job
+              items:
+                properties:
+                  name:
+                    description: Name specifies the name of tasks
+                    type: string
+                  policies:
+                    description: Specifies the lifecycle of task
+                    items:
+                      properties:
+                        action:
+                          description: The action that will be taken to the PodGroup
+                            according to Event. One of "Restart", "None". Default
+                            to None.
+                          type: string
+                        event:
+                          description: The Event recorded by scheduler; the controller
+                            takes actions according to this Event.
+                          type: string
+                        events:
+                          description: The Events recorded by scheduler; the controller takes
+                            actions according to this Events.
+                          type: array
+                          items:
+                            type: string
+                        timeout:
+                          description: Timeout is the grace period for controller
+                            to take actions. Default to nil (take action immediately).
+                          type: object
+                      type: object
+                    type: array
+                  replicas:
+                    description: Replicas specifies the replicas of this TaskSpec
+                      in Job
+                    format: int32
+                    type: integer
+                  template:
+                    description: Specifies the pod that will be created for this TaskSpec
+                      when executing a Job
+                    type: object
+                type: object
+              type: array
+            queue:
+              description: The name of the queue on which job should been created
+              type: string
+            maxRetry:
+              description: The limit for retrying submiting job, default is 3
+              format: int32
+              type: integer
+          type: object
+        status:
+          description: Current status of Job
+          properties:
+            succeeded:
+              description: The number of pods which reached phase Succeeded.
+              format: int32
+              type: integer
+            failed:
+              description: The number of pods which reached phase Failed.
+              format: int32
+              type: integer
+            minAvailable:
+              description: The minimal available pods to run for this Job
+              format: int32
+              type: integer
+            pending:
+              description: The number of pending pods.
+              format: int32
+              type: integer
+            running:
+              description: The number of running pods.
+              format: int32
+              type: integer
+            version:
+              description: Job's current version
+              format: int32
+              type: integer
+            retryCount:
+              description: The number that volcano retried to submit the job.
+              format: int32
+              type: integer
+            controlledResources:
+              description: All of the resources that are controlled by this job.
+              type: object
+              additionalProperties:
+                type: string
+            state:
+              description: Current state of Job.
+              properties:
+                message:
+                  description: Human-readable message indicating details about last
+                    transition.
+                  type: string
+                phase:
+                  description: The phase of Job
+                  type: string
+                reason:
+                  description: Unique, one-word, CamelCase reason for the condition's
+                    last transition.
+                  type: string
+                lastTransitionTime:
+                  description: The time of last state transition.
+                  format: date-time
+                  type: string
+              type: object
+          type: object
+  version: v1alpha1
+  subresources:
+    status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+
+---
+# Source: volcano/templates/bus_v1alpha1_command.yaml
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: commands.bus.volcano.sh
+spec:
+  group: bus.volcano.sh
+  names:
+    kind: Command
+    plural: commands
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      type: object
+      properties:
+        action:
+          description: Action defines the action that will be took to the target object.
+          type: string
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        message:
+          description: Human-readable message indicating details of this command.
+          type: string
+        metadata:
+          type: object
+        reason:
+          description: Unique, one-word, CamelCase reason for this command.
+          type: string
+        target:
+          description: TargetObject defines the target object of this command.
+          type: object
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+
+---
+# Source: volcano/templates/scheduling_v1beta1_podgroup.yaml
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: podgroups.scheduling.volcano.sh
+spec:
+  group: scheduling.volcano.sh
+  names:
+    kind: PodGroup
+    plural: podgroups
+    shortNames:
+      - pg
+      - podgroup-v1beta1
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          type: string
+        kind:
+          type: string
+        metadata:
+          type: object
+        spec:
+          properties:
+            minMember:
+              format: int32
+              type: integer
+            queue:
+              type: string
+            priorityClassName:
+              type: string
+          type: object
+        status:
+          properties:
+            succeeded:
+              format: int32
+              type: integer
+            failed:
+              format: int32
+              type: integer
+            running:
+              format: int32
+              type: integer
+          type: object
+      type: object
+  version: v1beta1
+
+---
+# Source: volcano/templates/scheduling_v1beta1_queue.yaml
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: queues.scheduling.volcano.sh
+spec:
+  group: scheduling.volcano.sh
+  names:
+    kind: Queue
+    plural: queues
+    shortNames:
+      - q
+      - queue-v1beta1
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          type: string
+        kind:
+          type: string
+        metadata:
+          type: object
+        spec:
+          properties:
+            weight:
+              format: int32
+              type: integer
+            capability:
+              type: object
+          type: object
+        status:
+          properties:
+            state:
+              type: string
+            unknown:
+              format: int32
+              type: integer
+            pending:
+              format: int32
+              type: integer
+            running:
+              format: int32
+              type: integer
+            inqueue:
+              format: int32
+              type: integer
+          type: object
+      type: object
+  version: v1beta1
+  subresources:
+    status: {}
+