From ed294f4c232ee4c7b6adf82c5c14035557bce1a3 Mon Sep 17 00:00:00 2001 From: Qstar Date: Fri, 6 Dec 2019 14:45:03 +0800 Subject: [PATCH] Ray Kubernetes Operator Part 1: readme, structure, config and CRD realted file (#6332) * Ray-Operator first PR 1.RayCluster CRD and CR, structure code in golang 2.config file in Kubernetes * Delete go.sum * Ray-Operator first PR 1.add directory structure 2.add guide for submitting RayCluster * Delete ray_v1_raycluster.bk.yaml * Ray-Operator first PR 1.delete file bk 2.add more description about kubernetes and ray-operator features * Ray-Operator first PR: adjust grammar * Ray-Operator first PR: add More Information about proposal * Ray-Operator first PR: 1.add heterogeneous version of CR 2.add reference ot key words, and reference links to the props in yaml 3.file structure to yaml level and function description * Ray-Operator first PR: add ray operator proposal doc * Ray-Operator first PR: add More Information about proposal * Ray-Operator first PR: add command to start * Ray-Operator first PR: add More Information about proposal * Update deploy/ray-operator/README.md Co-Authored-By: Edward Oakes * Update deploy/ray-operator/api/v1/raycluster_types.go Co-Authored-By: Edward Oakes * Update deploy/ray-operator/api/v1/raycluster_types.go Co-Authored-By: Edward Oakes * Ray-Operator first PR: add More Information about proposal * Ray-Operator first PR: remove License * Ray-Operator first PR: rename version from v1 to v1alpha1 * Ray-Operator first PR: use replicas instead of numNodes * Ray-Operator first PR: update replicas in CR yaml file * Ray-Operator first PR: add More Information about proposal --- deploy/ray-operator/README.md | 129 + .../api/v1alpha1/groupversion_info.go | 20 + .../api/v1alpha1/raycluster_types.go | 137 + .../api/v1alpha1/zz_generated.deepcopy.go | 192 ++ .../config/certmanager/certificate.yaml | 25 + .../config/certmanager/kustomization.yaml | 5 + .../config/certmanager/kustomizeconfig.yaml | 16 + .../config/crd/bases/ray.io_rayclusters.yaml | 2204 +++++++++++++++++ .../config/crd/kustomization.yaml | 21 + .../config/crd/kustomizeconfig.yaml | 17 + .../patches/cainjection_in_rayclusters.yaml | 8 + .../crd/patches/webhook_in_rayclusters.yaml | 17 + .../config/default/kustomization.yaml | 74 + .../default/manager_auth_proxy_patch.yaml | 25 + .../config/default/manager_webhook_patch.yaml | 23 + .../default/webhookcainjection_patch.yaml | 15 + .../config/manager/kustomization.yaml | 2 + .../ray-operator/config/manager/manager.yaml | 39 + .../config/prometheus/kustomization.yaml | 2 + .../config/prometheus/monitor.yaml | 15 + .../config/rbac/auth_proxy_role.yaml | 13 + .../config/rbac/auth_proxy_role_binding.yaml | 12 + .../config/rbac/auth_proxy_service.yaml | 14 + .../config/rbac/kustomization.yaml | 11 + .../config/rbac/leader_election_role.yaml | 32 + .../rbac/leader_election_role_binding.yaml | 12 + .../config/rbac/role_binding.yaml | 12 + .../samples/ray_v1_raycluster.complete.yaml | 260 ++ .../ray_v1_raycluster.heterogeneous.yaml | 146 ++ .../samples/ray_v1_raycluster.mini.yaml | 74 + .../config/webhook/kustomization.yaml | 6 + .../config/webhook/kustomizeconfig.yaml | 25 + .../config/webhook/manifests.yaml | 0 .../ray-operator/config/webhook/service.yaml | 12 + deploy/ray-operator/hack/boilerplate.go.txt | 0 35 files changed, 3615 insertions(+) create mode 100644 deploy/ray-operator/README.md create mode 100644 deploy/ray-operator/api/v1alpha1/groupversion_info.go create mode 100644 deploy/ray-operator/api/v1alpha1/raycluster_types.go create mode 100644 deploy/ray-operator/api/v1alpha1/zz_generated.deepcopy.go create mode 100644 deploy/ray-operator/config/certmanager/certificate.yaml create mode 100644 deploy/ray-operator/config/certmanager/kustomization.yaml create mode 100644 deploy/ray-operator/config/certmanager/kustomizeconfig.yaml create mode 100644 deploy/ray-operator/config/crd/bases/ray.io_rayclusters.yaml create mode 100644 deploy/ray-operator/config/crd/kustomization.yaml create mode 100644 deploy/ray-operator/config/crd/kustomizeconfig.yaml create mode 100644 deploy/ray-operator/config/crd/patches/cainjection_in_rayclusters.yaml create mode 100644 deploy/ray-operator/config/crd/patches/webhook_in_rayclusters.yaml create mode 100644 deploy/ray-operator/config/default/kustomization.yaml create mode 100644 deploy/ray-operator/config/default/manager_auth_proxy_patch.yaml create mode 100644 deploy/ray-operator/config/default/manager_webhook_patch.yaml create mode 100644 deploy/ray-operator/config/default/webhookcainjection_patch.yaml create mode 100644 deploy/ray-operator/config/manager/kustomization.yaml create mode 100644 deploy/ray-operator/config/manager/manager.yaml create mode 100644 deploy/ray-operator/config/prometheus/kustomization.yaml create mode 100644 deploy/ray-operator/config/prometheus/monitor.yaml create mode 100644 deploy/ray-operator/config/rbac/auth_proxy_role.yaml create mode 100644 deploy/ray-operator/config/rbac/auth_proxy_role_binding.yaml create mode 100644 deploy/ray-operator/config/rbac/auth_proxy_service.yaml create mode 100644 deploy/ray-operator/config/rbac/kustomization.yaml create mode 100644 deploy/ray-operator/config/rbac/leader_election_role.yaml create mode 100644 deploy/ray-operator/config/rbac/leader_election_role_binding.yaml create mode 100644 deploy/ray-operator/config/rbac/role_binding.yaml create mode 100644 deploy/ray-operator/config/samples/ray_v1_raycluster.complete.yaml create mode 100644 deploy/ray-operator/config/samples/ray_v1_raycluster.heterogeneous.yaml create mode 100644 deploy/ray-operator/config/samples/ray_v1_raycluster.mini.yaml create mode 100644 deploy/ray-operator/config/webhook/kustomization.yaml create mode 100644 deploy/ray-operator/config/webhook/kustomizeconfig.yaml create mode 100644 deploy/ray-operator/config/webhook/manifests.yaml create mode 100644 deploy/ray-operator/config/webhook/service.yaml create mode 100644 deploy/ray-operator/hack/boilerplate.go.txt diff --git a/deploy/ray-operator/README.md b/deploy/ray-operator/README.md new file mode 100644 index 000000000..6bd349d0a --- /dev/null +++ b/deploy/ray-operator/README.md @@ -0,0 +1,129 @@ +# Ray Kubernetes Operator + +The Ray Operator is a Kubernetes operator to automate provisioning, management, autoscaling and operations of Ray clusters deployed to Kubernetes. + +Some of the main features of Ray-Operator are: +- user management via CRD +- heterogeneous pods in one Ray cluster with specific affinity, toleration and other pre-defined settings +- monitoring via Prometheus +- HA for Ray Kubernetes Operator, there will be a lead election if lead crashes + +## File structure: +> ``` +> ray/deploy/ray-operator +> ├── api/v1alpha1 // Package v1alpha1 contains API Schema definitions for the ray v1alpha1 API group +> │ ├── groupversion_info.go // contains common metadata about the group-version +> │ ├── raycluster_types.go // RayCluster field definitions, user should focus +> │ └── zz_generated.deepcopy.go // contains the autogenerated implementation of the aforementioned runtime.Object interface, which marks all of our root types as representing Kinds. +> │ +> └── config // contains Kustomize YAML definitions required to launch our controller on a cluster,hold our CustomResourceDefinitions, RBAC configuration, and WebhookConfigurations. +> ├── certmanager +> │ ├── certificate.yaml // The following manifests contain a self-signed issuer CR and a certificate CR. +> │ ├── kustomization.yaml +> │ └── kustomizeconfig.yaml +> │ +> ├── crd +> │ └── bases +> │ │ └── ray.io_rayclusters.yaml // RayCluster CRD yaml file +> │ └── patches +> │ │ ├── cainjection_in_rayclusters.yaml // adds a directive for certmanager to inject CA into the CRD +> │ │ └── webhook_in_rayclusters.yaml // enables conversion webhook for CRD +> │ │── kustomization.yaml +> │ └── kustomizeconfig.yaml +> │ +> ├── default // contains a Kustomize base for launching the controller in a standard configuration. +> │ ├── kustomization.yaml +> │ ├── manager_auth_proxy_patch.yaml // inject a sidecar container which is a HTTP proxy for the controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +> │ ├── manager_webhook_patch.yaml // webhook yaml file +> │ └── webhookcainjection_patch.yaml // add annotation to admission webhook config +> │ +> ├── manager // launch your controllers as pods in the cluster. +> │ ├── kustomization.yaml +> │ └── manager.yaml // manager yaml to create controller deployment, user should focus +> │ +> ├── prometheus +> │ ├── kustomization.yaml +> │ └── monitor.yaml // Prometheus Monitor Service, user should focus +> │ +> ├── rbac // permissions required to run your controllers under their own service account. +> │ ├── auth_proxy_role.yaml +> │ ├── auth_proxy_role_binding.yaml +> │ ├── auth_proxy_service.yaml +> │ ├── kustomization.yaml +> │ ├── leader_election_role.yaml // permissions to do leader election. +> │ ├── leader_election_role_binding.yaml +> │ └── role_binding.yaml +> │ +> ├── samples // sample RayCluster yaml, user should focus +> │ ├── ray_v1_raycluster.complete.yaml +> │ ├── ray_v1_raycluster.heterogeneous.yaml +> │ └── ray_v1_raycluster.mini.yaml +> │ +> └── webhook +> ├── kustomization.yaml +> ├── kustomizeconfig.yaml +> ├── manifests.yaml +> └── service.yaml // webhook-service +> ``` + +## RayCluster sample CR + +To introduce the Ray-Operator, give 3 samples of RayCluster CR. + +Sample | desc +------------- | ------------- +[RayCluster.mini.yaml](config/samples/ray_v1_raycluster.mini.yaml) | 2 pods in this sample, 1 for head and 1 for workers.The least information to start ray cluster, run in local test. +[RayCluster.heterogeneous.yaml](config/samples/ray_v1_raycluster.heterogeneous.yaml) | 3 pods in this sample, 1 for head and 2 for workers but with different specifications. Different quota(like CPU/MEMORY) compares to mini version, run in local test. +[RayCluster.complete.yaml](config/samples/ray_v1_raycluster.complete.yaml) | a complete version CR for customized requirement, show how to set Customized props. More props to set compares to heterogeneous version, run in production. + +## RayCluster CRD + +Refers to file [raycluster_types.go](api/v1alpha1/raycluster_types.go) for code details. + +If interested in CRD, refer to file [CRD](config/crd/bases/ray.io_rayclusters.yaml) for more details. + +## Software requirement +Take care some software have dependency. + +software | version | memo +:------------- | :---------------:| -------------: +kustomize | v3.1.0+ | [download](https://github.com/kubernetes-sigs/kustomize) +kubectl | v1.11.3+ | [download](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +Kubernetes Cluster | Access to a Kubernetes v1.11.3+ cluster| [Minikube](https://github.com/kubernetes/minikube) for local test +go | v1.13+|[download](https://golang.org/dl/) +docker | 17.03+|[download](https://docs.docker.com/install/) + +Also you will need kubeconfig in ~/.kube/config, so you can access to Kubernetes Cluster. + +## Get started +Below gives a guide for user to submit RayCluster step by step: + +### Install CRDs into a cluster + +```shell script +kustomize build config/crd | kubectl apply -f - +``` + +### Deploy controller in the configured Kubernetes cluster in ~/.kube/config +* For this version controller will run in system namespace, which maybe can't be tolerated in production. +* We will add more detailed RBAC file to control the namespace used in production, and the controller will run in that namespace to control the permission. +* Also we will provide the more detailed guide for user to run in a controlled way. +```shell script +cd config/manager +kustomize build config/default | kubectl apply -f - +``` + +### Submit RayCluster to Kubernetes +```shell script +kubectl create -f config/samples/ray_v1_raycluster.mini.yaml +``` + +### Apply RayCluster to Kubernetes +```shell script +kubectl apply -f config/samples/ray_v1_raycluster.mini.yaml +``` + +### Delete RayCluster to Kubernetes +```shell script +kubectl delete -f config/samples/ray_v1_raycluster.mini.yaml +``` diff --git a/deploy/ray-operator/api/v1alpha1/groupversion_info.go b/deploy/ray-operator/api/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..426a297a1 --- /dev/null +++ b/deploy/ray-operator/api/v1alpha1/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1alpha1 contains API Schema definitions for the ray v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=ray.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "ray.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/deploy/ray-operator/api/v1alpha1/raycluster_types.go b/deploy/ray-operator/api/v1alpha1/raycluster_types.go new file mode 100644 index 000000000..09e2ccd76 --- /dev/null +++ b/deploy/ray-operator/api/v1alpha1/raycluster_types.go @@ -0,0 +1,137 @@ +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// RayClusterSpec defines the desired state of RayCluster +type RayClusterSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + // ClusterName is unique identifier for RayCluster in one namespace. + ClusterName string `json:"clusterName"` + // Docker image. + Images RayClusterImage `json:"images"` + // Image pull policy. + // One of Always, Never, IfNotPresent. + // Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + // Cannot be updated. + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` + // Specification of the desired behavior of pod group + Extensions []Extension `json:"extensions,omitempty"` +} + +// RayCluster pod type +type ReplicaType string + +const ( + // ReplicaTypeHead means that this pod will be ray cluster head + ReplicaTypeHead ReplicaType = "head" + // ReplicaTypeWorker means that this pod will be ray cluster worker + ReplicaTypeWorker ReplicaType = "worker" +) + +// RayClusterStatus defines the observed state of RayCluster +type RayClusterStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// +kubebuilder:object:root=true + +// RayCluster is the Schema for the RayClusters API +type RayCluster struct { + // Standard object metadata. + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the RayCluster. + Spec RayClusterSpec `json:"spec,omitempty"` + Status RayClusterStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// RayClusterList contains a list of RayCluster +type RayClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RayCluster `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RayCluster{}, &RayClusterList{}) +} + +type RayClusterImage struct { + // Default docker image + DefaultImage string `json:"defaultImage,omitempty"` +} + +type Extension struct { + // Number of desired pods in this pod group. This is a pointer to distinguish between explicit + // zero and not specified. Defaults to 1. + Replicas *int32 `json:"replicas"` + + // The pod type in this group worker/head. + Type ReplicaType `json:"type,omitempty"` + + // Pod docker image + Image string `json:"image,omitempty"` + + // Logical groupName for worker in same group, it's used for heterogeneous feature to distinguish different groups. + // GroupName is the unique identifier for the pods with the same configuration in one Ray Cluster + GroupName string `json:"groupName,omitempty"` + + // Command to start ray + Command string `json:"command,omitempty"` + + // Labels for pod, raycluster.component and rayclusters.ray.io/component-name are default labels, do not overwrite them. + // Labels are intended to be used to specify identifying attributes of objects that are meaningful and relevant to users, + // but do not directly imply semantics to the core system. Labels can be used to organize and to select subsets of objects. + Labels map[string]string `json:"labels,omitempty"` + + // NodeSelector specifies a map of key-value pairs. For the pod to be eligible + // to run on a node, the node must have each of the indicated key-value pairs as + // labels. Optional. + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Affinity is a group of affinity scheduling rules. Optional. + // It allows you to constrain which nodes your pod is eligible to be scheduled on, based on labels on the node. + Affinity *v1.Affinity `json:"affinity,omitempty"` + + // ResourceRequirements describes the compute resource requirements. + Resources v1.ResourceRequirements `json:"resources,omitempty"` + + // The pod this Toleration is attached to tolerates any taint that matches + // the triple using the matching operator . Optional. + // Tolerations are applied to pods, and allow (but do not require) the pods to schedule onto nodes with matching taints. + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + + // EnvVar represents an environment variable present in a Container. Optional. + // Define environment variables for a container. This can be helpful in defining variables before setting up the container. + ContainerEnv []v1.EnvVar `json:"containerEnv,omitempty"` + + // Head service suffix. So head can be accessed by domain name: {namespace}.svc , follows Kubernetes standard. + HeadServiceSuffix string `json:"headServiceSuffix,omitempty"` + + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. Optional. + // Usage refers to https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + Annotations map[string]string `json:"annotations,omitempty"` + + // Volume represents a named volume in a pod that may be accessed by any container in the pod. Optional. + // At its core, a volume is just a directory, possibly with some data in it, which is accessible to the Containers in a Pod. + Volumes []v1.Volume `json:"volumes,omitempty"` + + // VolumeMount describes a mounting of a Volume within a container. Optional. + VolumeMounts []v1.VolumeMount `json:"volumeMounts,omitempty"` +} diff --git a/deploy/ray-operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/ray-operator/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..6b6d291b3 --- /dev/null +++ b/deploy/ray-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,192 @@ +// +build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Extension) DeepCopyInto(out *Extension) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(corev1.Affinity) + (*in).DeepCopyInto(*out) + } + in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ContainerEnv != nil { + in, out := &in.ContainerEnv, &out.ContainerEnv + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]corev1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]corev1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Extension. +func (in *Extension) DeepCopy() *Extension { + if in == nil { + return nil + } + out := new(Extension) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayCluster) DeepCopyInto(out *RayCluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayCluster. +func (in *RayCluster) DeepCopy() *RayCluster { + if in == nil { + return nil + } + out := new(RayCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayCluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterImage) DeepCopyInto(out *RayClusterImage) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterImage. +func (in *RayClusterImage) DeepCopy() *RayClusterImage { + if in == nil { + return nil + } + out := new(RayClusterImage) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterList) DeepCopyInto(out *RayClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RayCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterList. +func (in *RayClusterList) DeepCopy() *RayClusterList { + if in == nil { + return nil + } + out := new(RayClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterSpec) DeepCopyInto(out *RayClusterSpec) { + *out = *in + out.Images = in.Images + if in.Extensions != nil { + in, out := &in.Extensions, &out.Extensions + *out = make([]Extension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterSpec. +func (in *RayClusterSpec) DeepCopy() *RayClusterSpec { + if in == nil { + return nil + } + out := new(RayClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterStatus) DeepCopyInto(out *RayClusterStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterStatus. +func (in *RayClusterStatus) DeepCopy() *RayClusterStatus { + if in == nil { + return nil + } + out := new(RayClusterStatus) + in.DeepCopyInto(out) + return out +} diff --git a/deploy/ray-operator/config/certmanager/certificate.yaml b/deploy/ray-operator/config/certmanager/certificate.yaml new file mode 100644 index 000000000..237c9378d --- /dev/null +++ b/deploy/ray-operator/config/certmanager/certificate.yaml @@ -0,0 +1,25 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for breaking changes +apiVersion: cert-manager.io/v1alpha2 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/deploy/ray-operator/config/certmanager/kustomization.yaml b/deploy/ray-operator/config/certmanager/kustomization.yaml new file mode 100644 index 000000000..bebea5a59 --- /dev/null +++ b/deploy/ray-operator/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/deploy/ray-operator/config/certmanager/kustomizeconfig.yaml b/deploy/ray-operator/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 000000000..90d7c313c --- /dev/null +++ b/deploy/ray-operator/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/deploy/ray-operator/config/crd/bases/ray.io_rayclusters.yaml b/deploy/ray-operator/config/crd/bases/ray.io_rayclusters.yaml new file mode 100644 index 000000000..eed593ff0 --- /dev/null +++ b/deploy/ray-operator/config/crd/bases/ray.io_rayclusters.yaml @@ -0,0 +1,2204 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.2 + creationTimestamp: null + name: rayclusters.ray.io +spec: + group: ray.io + names: + kind: RayCluster + listKind: RayClusterList + plural: rayclusters + singular: raycluster + scope: "" + validation: + openAPIV3Schema: + description: RayCluster is the Schema for the RayClusters API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of the RayCluster. + properties: + clusterName: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + ClusterName is unique identifier for RayCluster in one namespace.' + type: string + extensions: + description: Specification of the desired behavior of pod group + items: + properties: + affinity: + description: Affinity is a group of affinity scheduling rules. + Optional. It allows you to constrain which nodes your pod is + eligible to be scheduled on, based on labels on the node. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a no-op). + A null preferred scheduling term matches no objects + (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from + its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them are + ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to a pod label update), + the system may or may not try to eventually evict the + pod from its node. When there are multiple elements, + the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node that + violates one or more of the expressions. The node that + is most preferred is the one with the greatest sum of + weights, i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + anti-affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the pod + will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod + label update), the system may or may not try to eventually + evict the pod from its node. When there are multiple + elements, the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + description: Annotations is an unstructured key value map stored + with a resource that may be set by external tools to store and + retrieve arbitrary metadata. They are not queryable and should + be preserved when modifying objects. Optional. Usage refers + to https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + type: object + command: + description: Command to start ray + type: string + containerEnv: + description: EnvVar represents an environment variable present + in a Container. Optional. Define environment variables for a + container. This can be helpful in defining variables before + setting up the container. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a + variable cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax can be + escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + groupName: + description: Logical groupName for worker in same group, it's + used for heterogeneous feature to distinguish different groups. + GroupName is the unique identifier for the pods with the same + configuration in one Ray Cluster + type: string + headServiceSuffix: + description: 'Head service suffix. So head can be accessed by + domain name: {namespace}.svc , follows Kubernetes standard.' + type: string + image: + description: Pod docker image + type: string + labels: + additionalProperties: + type: string + description: Labels for pod, raycluster.component and rayclusters.ray.io/component-name + are default labels, do not overwrite them. Labels are intended + to be used to specify identifying attributes of objects that + are meaningful and relevant to users, but do not directly imply + semantics to the core system. Labels can be used to organize + and to select subsets of objects. + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector specifies a map of key-value pairs. + For the pod to be eligible to run on a node, the node must have + each of the indicated key-value pairs as labels. Optional. + type: object + replicas: + description: Number of desired pods in this pod group. This is + a pointer to distinguish between explicit zero and not specified. + Defaults to 1. + format: int32 + type: integer + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + tolerations: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using the + matching operator . Optional. Tolerations are applied + to pods, and allow (but do not require) the pods to schedule + onto nodes with matching taints. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, allowed + values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match + all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to + the value. Valid operators are Exists and Equal. Defaults + to Equal. Exists is equivalent to wildcard for value, + so that a pod can tolerate all taints of a particular + category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the taint + forever (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + type: + description: The pod type in this group worker/head. + type: string + volumeMounts: + description: VolumeMount describes a mounting of a Volume within + a container. Optional. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. This field is beta in 1.15. + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + description: Volume represents a named volume in a pod that may + be accessed by any container in the pod. Optional. At its core, + a volume is just a directory, possibly with some data in it, + which is accessible to the Containers in a Pod. + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'The partition in the volume that you want + to mount. If omitted, the default is to mount by volume + name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property + empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the ReadOnly + property in VolumeMounts to "true". If omitted, the + default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource + in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: The Name of the data disk in the blob storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob + disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure + Storage Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of + Ceph monitors More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, rather + than the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to key + ring for User, default is /etc/ceph/user.secret More + info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to the + authentication secret for User, default is empty. + More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, + default is admin More info: https://releases.k8s.io/HEAD/examples/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached + and mounted on kubelets host machine More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object containing + parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume + in cinder More info: https://releases.k8s.io/HEAD/examples/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and 0777. + Defaults to 0644. Directories within the path are + not affected by this setting. This might be in conflict + with other options that affect the file mode, like + fsGroup, and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced ConfigMap will be + projected into the volume as a file whose name is + the key and content is the value. If specified, the + listed keys will be projected into the specified paths, + and unlisted keys will not be present. If a key is + specified which is not present in the ConfigMap, the + volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this + file, must be a value between 0 and 0777. If + not specified, the volume defaultMode will be + used. This might be in conflict with other options + that affect the file mode, like fsGroup, and + the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its keys + must be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents + storage that is handled by an external CSI driver (Alpha + feature). + properties: + driver: + description: Driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed + to the associated CSI driver which will determine + the default filesystem to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to + the secret object containing sensitive information + to pass to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the + secret object contains more than one secret, all secret + references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for + the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. Consult + your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and 0777. + Defaults to 0644. Directories within the path are + not affected by this setting. This might be in conflict + with other options that affect the file mode, like + fsGroup, and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on this + file, must be a value between 0 and 0777. If + not specified, the volume defaultMode will be + used. This might be in conflict with other options + that affect the file mode, like fsGroup, and + the result can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must not + be absolute or contain the ''..'' path. Must + be utf-8 encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + description: Specifies the output format of + the exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back + this directory. The default is "" which means to use + the node''s default medium. Must be an empty string + (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + description: 'Total amount of local storage required + for this EmptyDir volume. The size limit is also applicable + for memory medium. The maximum usage on memory medium + EmptyDir would be the minimum value between the SizeLimit + specified here and the sum of memory limits of all + containers in a pod. The default is nil which means + that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + type: string + type: object + fc: + description: FC represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. TODO: how do we prevent errors in the + filesystem from compromising the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use + for this volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends on FlexVolume + script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to the + secret object containing sensitive information to + pass to the plugin scripts. This may be empty if no + secret object is specified. If the secret object contains + more than one secret, all secrets are passed to the + plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: Name of the dataset stored as metadata + -> name on the dataset for Flocker should be considered + as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then + exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'The partition in the volume that you want + to mount. If omitted, the default is to mount by volume + name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property + empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. + Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an + InitContainer that clones the repo using git, then mount + the EmptyDir into the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain + or start with '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on + the host that shares a pod''s lifetime. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that + details Glusterfs topology. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More + info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs + volume to be mounted with read-only permissions. Defaults + to false. More info: https://releases.k8s.io/HEAD/examples/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file or + directory on the host machine that is directly exposed + to the container. This is generally used for system agents + or other privileged things that are allowed to see the + host machine. Most containers will NOT need this. More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host + directory mounts and who can/can not mount host directories + as read/write.' + properties: + path: + description: 'Path of the directory on the host. If + the path is a symlink, it will follow the link to + the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://releases.k8s.io/HEAD/examples/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, new + iSCSI interface : will + be created for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI + transport. Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal is + either an IP or ip_addr:port if the port is other + than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator + authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is either + an IP or ip_addr:port if the port is other than default + (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and unique + within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + pdID: + description: ID that identifies Photon Controller persistent + disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits to use on created files by default. + Must be a value between 0 and 0777. Directories within + the path are not affected by this setting. This might + be in conflict with other options that affect the + file mode, like fsGroup, and the result can be other + mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: information about the configMap data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to + use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to + use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + description: Specifies the output + format of the exposed resources, + defaults to "1" + type: string + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to + use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if + the token is older than 80 percent of its + time to live or if the token is older than + 24 hours.Defaults to 1 hour and must be + at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to + the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + required: + - sources + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default is + no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults + to false. + type: boolean + registry: + description: Registry represents a single or multiple + Quobyte Registry services specified as a string as + host:port pair (multiple entries are separated with + commas) which acts as the central registry for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: User to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: Volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'The rados image name. More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More info: + https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. More + info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. + More info: https://releases.k8s.io/HEAD/examples/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for + ScaleIO user and other sensitive information. If this + is not provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume + should be ThickProvisioned or ThinProvisioned. Default + is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with + the protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created in + the ScaleIO system that is associated with this volume + source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and 0777. + Defaults to 0644. Directories within the path are + not affected by this setting. This might be in conflict + with other options that affect the file mode, like + fsGroup, and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and + content is the value. If specified, the listed keys + will be projected into the specified paths, and unlisted + keys will not be present. If a key is specified which + is not present in the Secret, the volume setup will + error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start + with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this + file, must be a value between 0 and 0777. If + not specified, the volume defaultMode will be + used. This might be in conflict with other options + that affect the file mode, like fsGroup, and + the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys + must be defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace + to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of + the volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within + StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. Set + to "default" if you are not using namespaces within + StorageOS. Namespaces that do not pre-exist within + StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - replicas + type: object + type: array + imagePullPolicy: + description: Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + type: string + images: + description: Docker image name. + properties: + defaultImage: + description: Default docker image + type: string + type: object + required: + - clusterName + - images + type: object + status: + description: RayClusterStatus defines the observed state of RayCluster + type: object + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/ray-operator/config/crd/kustomization.yaml b/deploy/ray-operator/config/crd/kustomization.yaml new file mode 100644 index 000000000..778c09a04 --- /dev/null +++ b/deploy/ray-operator/config/crd/kustomization.yaml @@ -0,0 +1,21 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/ray.io_rayclusters.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_rayclusters.yaml +# +kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_rayclusters.yaml +# +kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/deploy/ray-operator/config/crd/kustomizeconfig.yaml b/deploy/ray-operator/config/crd/kustomizeconfig.yaml new file mode 100644 index 000000000..6f83d9a94 --- /dev/null +++ b/deploy/ray-operator/config/crd/kustomizeconfig.yaml @@ -0,0 +1,17 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + group: apiextensions.k8s.io + path: spec/conversion/webhookClientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/deploy/ray-operator/config/crd/patches/cainjection_in_rayclusters.yaml b/deploy/ray-operator/config/crd/patches/cainjection_in_rayclusters.yaml new file mode 100644 index 000000000..012a444f5 --- /dev/null +++ b/deploy/ray-operator/config/crd/patches/cainjection_in_rayclusters.yaml @@ -0,0 +1,8 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: rayclusters.ray.io diff --git a/deploy/ray-operator/config/crd/patches/webhook_in_rayclusters.yaml b/deploy/ray-operator/config/crd/patches/webhook_in_rayclusters.yaml new file mode 100644 index 000000000..e7e8438e1 --- /dev/null +++ b/deploy/ray-operator/config/crd/patches/webhook_in_rayclusters.yaml @@ -0,0 +1,17 @@ +# The following patch enables conversion webhook for CRD +# CRD conversion requires k8s 1.13 or later. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: rayclusters.ray.io +spec: + conversion: + strategy: Webhook + webhookClientConfig: + # this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank, + # but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager) + caBundle: Cg== + service: + namespace: system + name: webhook-service + path: /convert diff --git a/deploy/ray-operator/config/default/kustomization.yaml b/deploy/ray-operator/config/default/kustomization.yaml new file mode 100644 index 000000000..1197a885f --- /dev/null +++ b/deploy/ray-operator/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: ray-operator-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: ray-operator- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: + # Protect the /metrics endpoint by putting it behind auth. + # Only one of manager_auth_proxy_patch.yaml and + # manager_prometheus_metrics_patch.yaml should be enabled. +- manager_auth_proxy_patch.yaml + # If you want your controller-manager to expose the /metrics + # endpoint w/o any authn/z, uncomment the following line and + # comment manager_auth_proxy_patch.yaml. + # Only one of manager_auth_proxy_patch.yaml and + # manager_prometheus_metrics_patch.yaml should be enabled. +#- manager_prometheus_metrics_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1alpha2 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1alpha2 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/deploy/ray-operator/config/default/manager_auth_proxy_patch.yaml b/deploy/ray-operator/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 000000000..61cb5e7cb --- /dev/null +++ b/deploy/ray-operator/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,25 @@ +# This patch inject a sidecar container which is a HTTP proxy for the controller manager, +# it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + name: https + - name: manager + args: + - "--metrics-addr=127.0.0.1:8080" + - "--enable-leader-election" diff --git a/deploy/ray-operator/config/default/manager_webhook_patch.yaml b/deploy/ray-operator/config/default/manager_webhook_patch.yaml new file mode 100644 index 000000000..738de350b --- /dev/null +++ b/deploy/ray-operator/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/deploy/ray-operator/config/default/webhookcainjection_patch.yaml b/deploy/ray-operator/config/default/webhookcainjection_patch.yaml new file mode 100644 index 000000000..7e79bf995 --- /dev/null +++ b/deploy/ray-operator/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/deploy/ray-operator/config/manager/kustomization.yaml b/deploy/ray-operator/config/manager/kustomization.yaml new file mode 100644 index 000000000..5c5f0b84c --- /dev/null +++ b/deploy/ray-operator/config/manager/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- manager.yaml diff --git a/deploy/ray-operator/config/manager/manager.yaml b/deploy/ray-operator/config/manager/manager.yaml new file mode 100644 index 000000000..b6c85a52d --- /dev/null +++ b/deploy/ray-operator/config/manager/manager.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - command: + - /manager + args: + - --enable-leader-election + image: controller:latest + name: manager + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + terminationGracePeriodSeconds: 10 diff --git a/deploy/ray-operator/config/prometheus/kustomization.yaml b/deploy/ray-operator/config/prometheus/kustomization.yaml new file mode 100644 index 000000000..ed137168a --- /dev/null +++ b/deploy/ray-operator/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/deploy/ray-operator/config/prometheus/monitor.yaml b/deploy/ray-operator/config/prometheus/monitor.yaml new file mode 100644 index 000000000..e2d9b087f --- /dev/null +++ b/deploy/ray-operator/config/prometheus/monitor.yaml @@ -0,0 +1,15 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + selector: + control-plane: controller-manager diff --git a/deploy/ray-operator/config/rbac/auth_proxy_role.yaml b/deploy/ray-operator/config/rbac/auth_proxy_role.yaml new file mode 100644 index 000000000..618f5e417 --- /dev/null +++ b/deploy/ray-operator/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: ["authentication.k8s.io"] + resources: + - tokenreviews + verbs: ["create"] +- apiGroups: ["authorization.k8s.io"] + resources: + - subjectaccessreviews + verbs: ["create"] diff --git a/deploy/ray-operator/config/rbac/auth_proxy_role_binding.yaml b/deploy/ray-operator/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 000000000..48ed1e4b8 --- /dev/null +++ b/deploy/ray-operator/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/deploy/ray-operator/config/rbac/auth_proxy_service.yaml b/deploy/ray-operator/config/rbac/auth_proxy_service.yaml new file mode 100644 index 000000000..6cf656be1 --- /dev/null +++ b/deploy/ray-operator/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + targetPort: https + selector: + control-plane: controller-manager diff --git a/deploy/ray-operator/config/rbac/kustomization.yaml b/deploy/ray-operator/config/rbac/kustomization.yaml new file mode 100644 index 000000000..817f1fe61 --- /dev/null +++ b/deploy/ray-operator/config/rbac/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 3 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml diff --git a/deploy/ray-operator/config/rbac/leader_election_role.yaml b/deploy/ray-operator/config/rbac/leader_election_role.yaml new file mode 100644 index 000000000..eaa79158f --- /dev/null +++ b/deploy/ray-operator/config/rbac/leader_election_role.yaml @@ -0,0 +1,32 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/ray-operator/config/rbac/leader_election_role_binding.yaml b/deploy/ray-operator/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 000000000..eed16906f --- /dev/null +++ b/deploy/ray-operator/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/deploy/ray-operator/config/rbac/role_binding.yaml b/deploy/ray-operator/config/rbac/role_binding.yaml new file mode 100644 index 000000000..8f2658702 --- /dev/null +++ b/deploy/ray-operator/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: default + namespace: system diff --git a/deploy/ray-operator/config/samples/ray_v1_raycluster.complete.yaml b/deploy/ray-operator/config/samples/ray_v1_raycluster.complete.yaml new file mode 100644 index 000000000..6198738df --- /dev/null +++ b/deploy/ray-operator/config/samples/ray_v1_raycluster.complete.yaml @@ -0,0 +1,260 @@ +apiVersion: ray.io/v1alpha1 +kind: RayCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + # An unique identifier for the head node and workers of this cluster. + name: raycluster-sample +spec: + # An unique identifier for the head node and workers of this cluster. + clusterName: raycluster-sample + images: + defaultImage: "docker-image" + imagePullPolicy: "Always" + + extensions: + # the pod replicas in this group typed worker + - replicas: 1 + # logical group name, for this called small-group, also can be functional + groupName: small-group + # pod type + type: worker + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: small-group + + # annotations for pod + annotations: + key: value + + # use affinity to select nodes.Optional. + # Refer to https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: In + values: ["true"] + - key: key2 + operator: In + values: ["true"] + + # use tolerations to select nodes.Optional. + # Refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: + - key: "key1" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + # resource requirements. + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 1000m + memory: 2Gi + ephemeral-storage: 2Gi + requests: + cpu: 1000m + memory: 2Gi + ephemeral-storage: 2Gi + + # environment variables to set in the container.Optional. + # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + containerEnv: + - name: APPNAME + value: raycluster-sample + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" + + # use volumes.Optional. + # Refer to https://kubernetes.io/docs/concepts/storage/volumes/ + volumes: + - name: log-volume + emptyDir: {} + + # use volumeMounts.Optional. + # Refer to https://kubernetes.io/docs/concepts/storage/volumes/ + volumeMounts: + - mountPath: /path/to/log + name: log-volume + + # the pod replicas in this group typed worker + - replicas: 1 + # logical group name + groupName: medium-group + # pod type + type: worker + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: medium-group + + # annotations for pod + annotations: + key: value + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # use affinity to select nodes.Optional. + # Refer to https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: In + values: ["true"] + - key: key2 + operator: In + values: ["true"] + + # use tolerations to select nodes.Optional. + # Refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: + - key: "key1" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + # resource requirements. + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + requests: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + + # environment variables to set in the container.Optional. + # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + containerEnv: + - name: APPNAME + value: raycluster-sample + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" + + # use volumes.Optional. + # Refer to https://kubernetes.io/docs/concepts/storage/volumes/ + volumes: + - name: log-volume + emptyDir: {} + + # use volumeMounts.Optional. + # Refer to https://kubernetes.io/docs/concepts/storage/volumes/ + volumeMounts: + - mountPath: /path/to/log + name: log-volume + + # the pod replicas in this group typed head + - replicas: 1 + # logical group name + groupName: headgroup + # pod type + type: head + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: headgroup + + # annotations for pod + annotations: + key: value + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # use affinity to select nodes.Optional. + # Refer to https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: In + values: ["true"] + - key: key2 + operator: In + values: ["true"] + + # use tolerations to select nodes.Optional. + # Refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: + - key: "key1" + operator: "Equal" + value: "true" + effect: "NoSchedule" + + # resource requirements. + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 1000m + memory: 1Gi + ephemeral-storage: 4Gi + requests: + cpu: 1000m + memory: 1Gi + ephemeral-storage: 4Gi + + # environment variables to set in the container.Optional. + # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + containerEnv: + - name: APPNAME + value: raycluster-sample + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" + + # use volumes.Optional. + # Refer to https://kubernetes.io/docs/concepts/storage/volumes/ + volumes: + - name: log-volume + emptyDir: {} + + # use volumeMounts.Optional. + # Refer to https://kubernetes.io/docs/concepts/storage/volumes/ + volumeMounts: + - mountPath: /path/to/log + name: log-volume diff --git a/deploy/ray-operator/config/samples/ray_v1_raycluster.heterogeneous.yaml b/deploy/ray-operator/config/samples/ray_v1_raycluster.heterogeneous.yaml new file mode 100644 index 000000000..21445ecad --- /dev/null +++ b/deploy/ray-operator/config/samples/ray_v1_raycluster.heterogeneous.yaml @@ -0,0 +1,146 @@ +apiVersion: ray.io/v1alpha1 +kind: RayCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + # An unique identifier for the head node and workers of this cluster. + name: raycluster-sample +spec: + # An unique identifier for the head node and workers of this cluster. + clusterName: raycluster-sample + images: + defaultImage: "docker-image" + imagePullPolicy: "Always" + + extensions: + # the pod replicas in this group typed worker + - replicas: 1 + # logical group name, for this called small-group, also can be functional + groupName: small-group + # pod type + type: worker + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: small-group + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # resource requirements + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 1000m + memory: 2Gi + ephemeral-storage: 2Gi + requests: + cpu: 1000m + memory: 2Gi + ephemeral-storage: 2Gi + + # environment variables to set in the container.Optional + # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + containerEnv: + - name: CLUSTER_NAME + value: raycluster-sample + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" + + # the pod replicas in this group typed worker + - replicas: 1 + # logical group name, for this called medium-group, also can be functional + groupName: medium-group + # pod type + type: worker + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: medium-group + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # resource requirements + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + requests: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + + # environment variables to set in the container.Optional + # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + containerEnv: + - name: CLUSTER_NAME + value: raycluster-sample + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" + + # the pod replicas in this group typed head + - replicas: 1 + # logical group name, also can be functional + groupName: head-group + # pod type + type: head + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: head-group + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # resource requirements + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + requests: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + + # environment variables to set in the container.Optional + # Refer to https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/ + containerEnv: + - name: CLUSTER_NAME + value: raycluster-sample + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" diff --git a/deploy/ray-operator/config/samples/ray_v1_raycluster.mini.yaml b/deploy/ray-operator/config/samples/ray_v1_raycluster.mini.yaml new file mode 100644 index 000000000..edd1f830c --- /dev/null +++ b/deploy/ray-operator/config/samples/ray_v1_raycluster.mini.yaml @@ -0,0 +1,74 @@ +apiVersion: ray.io/v1alpha1 +kind: RayCluster +metadata: + labels: + controller-tools.k8s.io: "1.0" + # An unique identifier for the head node and workers of this cluster. + name: raycluster-sample +spec: + # An unique identifier for the head node and workers of this cluster. + clusterName: raycluster-sample + images: + defaultImage: "docker-image" + imagePullPolicy: "Always" + + extensions: + # the pod replicas in this group typed worker + - replicas: 1 + # logical group name, for this called small-group, also can be functional like hello-group + groupName: small-group + # pod type + type: worker + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: small-group + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # resource requirements + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 1000m + memory: 2Gi + ephemeral-storage: 2Gi + requests: + cpu: 1000m + memory: 2Gi + ephemeral-storage: 2Gi + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" + + # the pod replicas in this group typed head + - replicas: 1 + # logical group name, also can be functional + groupName: head-group + # pod type + type: head + + # custom labels. NOTE: do not define custom labels start with `raycluster.`, they may be used in controller. + # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + labels: + raycluster.group.name: head-group + + # Command to start ray + command: ray stop; ulimit -n 65536; ray start --object-manager-port=8076 + + # resource requirements + # Refer to https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + resources: + limits: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + requests: + cpu: 2000m + memory: 4Gi + ephemeral-storage: 4Gi + + # head service suffix: {namespace}.svc , follows Kubernetes standard + headServiceSuffix: "ray-operator.svc" diff --git a/deploy/ray-operator/config/webhook/kustomization.yaml b/deploy/ray-operator/config/webhook/kustomization.yaml new file mode 100644 index 000000000..9cf26134e --- /dev/null +++ b/deploy/ray-operator/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/deploy/ray-operator/config/webhook/kustomizeconfig.yaml b/deploy/ray-operator/config/webhook/kustomizeconfig.yaml new file mode 100644 index 000000000..25e21e3c9 --- /dev/null +++ b/deploy/ray-operator/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/deploy/ray-operator/config/webhook/manifests.yaml b/deploy/ray-operator/config/webhook/manifests.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/ray-operator/config/webhook/service.yaml b/deploy/ray-operator/config/webhook/service.yaml new file mode 100644 index 000000000..31e0f8295 --- /dev/null +++ b/deploy/ray-operator/config/webhook/service.yaml @@ -0,0 +1,12 @@ + +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/deploy/ray-operator/hack/boilerplate.go.txt b/deploy/ray-operator/hack/boilerplate.go.txt new file mode 100644 index 000000000..e69de29bb