Ray operator: controller code and guide to use (#6501)

This commit is contained in:
Qstar
2019-12-29 10:14:47 -06:00
committed by Edward Oakes
parent 7c1e0e5715
commit 10338fde0c
17 changed files with 1424 additions and 59 deletions
@@ -0,0 +1,33 @@
package common
const (
// Head used as pod type to decide create service or not, for now only create service for head.
Head = "head"
// Belows used as label key
//rayclusterComponent is the pod name for this pod for selecting pod by pod name.
rayclusterComponent = "raycluster.component"
// rayIoComponent is the identifier for created by ray-operator for selecting pod by operator name.
rayIoComponent = "rayclusters.ray.io/component-name"
// RayClusterOwnerKey is the ray cluster instance name for selecting pod by instance name.
RayClusterOwnerKey = "raycluster.instance.name"
// ClusterPodType is the pod type label key for selecting pod by type.
ClusterPodType = "raycluster.pod.type"
// rayOperator is the value of ray-operator used as identifier for the pod
rayOperator = "ray-operator"
// Use as separator for pod name, for example, raycluster-small-size-worker-0
DashSymbol = "-"
// Use as default port
defaultHTTPServerPort = 30021
defaultRedisPort = 6379
// Check node if ready by checking the path exists or not
PodReadyFilepath = "POD_READY_FILEPATH"
// Use as container env variable
namespace = "NAMESPACE"
clusterName = "CLUSTER_NAME"
)
@@ -0,0 +1,18 @@
package common
import rayiov1alpha1 "ray-operator/api/v1alpha1"
// The function labelsForCluster returns the labels for selecting the resources
// belonging to the given RayCluster CR name.
func labelsForCluster(instance rayiov1alpha1.RayCluster, name string, podTypeName string, extend map[string]string) (ret map[string]string) {
ret = map[string]string{
rayclusterComponent: name,
rayIoComponent: rayOperator,
RayClusterOwnerKey: instance.Name,
ClusterPodType: podTypeName,
}
for k, v := range extend {
ret[k] = v
}
return
}
@@ -0,0 +1,129 @@
package common
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
rayiov1alpha1 "ray-operator/api/v1alpha1"
"strings"
)
type PodConfig struct {
RayCluster *rayiov1alpha1.RayCluster
PodTypeName string
PodName string
Extension rayiov1alpha1.Extension
}
func DefaultPodConfig(instance *rayiov1alpha1.RayCluster, podTypeName string, podName string) *PodConfig {
return &PodConfig{
RayCluster: instance,
PodTypeName: podTypeName,
PodName: podName,
}
}
// Build a pod for the cluster instance.
func BuildPod(conf *PodConfig) *corev1.Pod {
// build label for cluster
rayLabels := labelsForCluster(*conf.RayCluster, conf.PodName, conf.PodTypeName, conf.Extension.Labels)
// build container for pod, now only handle one container for each pod
var containers []corev1.Container
container := buildContainer(conf)
containers = append(containers, container)
// create volume
volumes := conf.Extension.Volumes
spec := corev1.PodSpec{
Volumes: volumes,
Containers: containers,
Affinity: conf.Extension.Affinity,
Tolerations: conf.Extension.Tolerations,
ServiceAccountName: conf.RayCluster.Namespace,
}
// build annotations and store podCompareHash for comparison
annotations := conf.Extension.Annotations
pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
Name: conf.PodName,
Namespace: conf.RayCluster.Namespace,
Labels: rayLabels,
Annotations: annotations,
},
Spec: spec,
}
return pod
}
// Build container for pod.
func buildContainer(conf *PodConfig) corev1.Container {
redisPort := defaultRedisPort
httpServerPort := defaultHTTPServerPort
jobManagerPort := defaultRedisPort
// get pod file path to check if the pod container ready or not
var podReadyFilepath string
for _, env := range conf.Extension.ContainerEnv {
if strings.EqualFold(env.Name, PodReadyFilepath) {
podReadyFilepath = env.Value
break
}
}
// assign image by typeName
image := conf.RayCluster.Spec.Images.DefaultImage
if conf.Extension.Image != "" {
image = conf.Extension.Image
}
volumeMounts := conf.Extension.VolumeMounts
// add instance name and namespace to container env to identify cluster pods
var containerEnv []corev1.EnvVar
containerEnv = conf.Extension.ContainerEnv
containerEnv = append(containerEnv,
corev1.EnvVar{Name: namespace, Value: conf.RayCluster.Namespace},
corev1.EnvVar{Name: clusterName, Value: conf.RayCluster.Name})
container := corev1.Container{
Name: strings.ToLower(conf.PodTypeName),
Image: image,
Command: []string{"/bin/bash", "-c", "--"},
Args: []string{conf.Extension.Command},
Env: containerEnv,
Resources: conf.Extension.Resources,
VolumeMounts: volumeMounts,
ImagePullPolicy: conf.RayCluster.Spec.ImagePullPolicy,
Ports: []corev1.ContainerPort{
{
ContainerPort: int32(redisPort),
Name: "redis",
},
{
ContainerPort: int32(httpServerPort),
Name: "http-server",
},
{
ContainerPort: int32(jobManagerPort),
Name: "job-manager",
},
},
ReadinessProbe: &corev1.Probe{
Handler: corev1.Handler{
Exec: &corev1.ExecAction{Command: []string{"cat", podReadyFilepath}},
},
InitialDelaySeconds: 15,
SuccessThreshold: 2,
},
}
return container
}
@@ -0,0 +1,45 @@
package common
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
rayiov1alpha1 "ray-operator/api/v1alpha1"
"ray-operator/controllers/utils"
"strings"
)
type ServiceConfig struct {
RayCluster rayiov1alpha1.RayCluster
PodName string
}
func DefaultServiceConfig(instance rayiov1alpha1.RayCluster, podName string) *ServiceConfig {
return &ServiceConfig{
RayCluster: instance,
PodName: podName,
}
}
// Build service for pod, for now only head pod will have service.
func ServiceForPod(conf *ServiceConfig) *corev1.Service {
name := conf.PodName
if strings.Contains(conf.PodName, Head) {
name = utils.Before(conf.PodName, Head) + "head"
}
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: conf.RayCluster.Namespace,
},
Spec: corev1.ServiceSpec{
ClusterIP: "None",
// select this raycluster's component
Selector: map[string]string{
rayclusterComponent: conf.PodName,
},
},
}
return svc
}
@@ -0,0 +1,251 @@
package controllers
import (
"context"
"fmt"
mapset "github.com/deckarep/golang-set"
"github.com/go-logr/logr"
_ "k8s.io/api/apps/v1beta1"
rayiov1alpha1 "ray-operator/api/v1alpha1"
"ray-operator/controllers/common"
_ "ray-operator/controllers/common"
"ray-operator/controllers/utils"
"strings"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
"sigs.k8s.io/controller-runtime/pkg/source"
)
var log = logf.Log.WithName("RayCluster-Controller")
// Add creates a new RayCluster Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
// and start it when the Manager Started.
func Add(mgr manager.Manager) error {
return add(mgr, newReconciler(mgr))
}
// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager) reconcile.Reconciler {
return &RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()}
}
// add creates a new Controller to mgr with r as the reconcile.Reconciler
func add(mgr manager.Manager, r reconcile.Reconciler) error {
// Create a new controller
c, err := controller.New("ray-operator-RayCluster-controller", mgr, controller.Options{Reconciler: r})
if err != nil {
return err
}
// Watch for changes to RayCluster
err = c.Watch(&source.Kind{Type: &rayiov1alpha1.RayCluster{}}, &handler.EnqueueRequestForObject{})
if err != nil {
return err
}
err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{
IsController: true,
OwnerType: &rayiov1alpha1.RayCluster{},
})
if err != nil {
return err
}
return nil
}
var _ reconcile.Reconciler = &RayClusterReconciler{}
// ReconcileRayCluster reconciles a RayCluster object
type RayClusterReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
}
// Reconcile reads that state of the cluster for a RayCluster object and makes changes based on it
// and what is in the RayCluster.Spec
// Automatically generate RBAC rules to allow the Controller to read and write workloads
// +kubebuilder:rbac:groups=ray.io,resources=RayClusters,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=ray.io,resources=RayClusters/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch
// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch
func (r *RayClusterReconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) {
_ = r.Log.WithValues("raycluster", request.NamespacedName)
log.Info("Reconciling RayCluster", "cluster name", request.Name)
// Fetch the RayCluster instance
instance := &rayiov1alpha1.RayCluster{}
err := r.Get(context.TODO(), request.NamespacedName, instance)
if err != nil {
if errors.IsNotFound(err) {
// Object not found, return. Created objects are automatically garbage collected.
// For additional cleanup logic use finalizers.
return reconcile.Result{}, nil
}
log.Error(err, "Read request instance error!")
// Error reading the object - requeue the request.
return reconcile.Result{}, ignoreNotFound(err)
}
log.Info("Print instance - ", "Instance.ToString", instance)
// Build pods for instance
expectedPods := r.buildPods(instance)
expectedPodNameList := mapset.NewSet()
expectedPodMap := make(map[string]corev1.Pod)
needServicePodMap := mapset.NewSet()
for _, pod := range expectedPods {
expectedPodNameList.Add(pod.Name)
expectedPodMap[pod.Name] = pod
if strings.EqualFold(pod.Labels[common.ClusterPodType], common.Head) {
needServicePodMap.Add(pod.Name)
}
}
log.Info("Build pods according to the ray cluster instance", "size", len(expectedPods), "podNames", expectedPodNameList)
runtimePods := corev1.PodList{}
if err = r.List(context.TODO(), &runtimePods, client.InNamespace(instance.Namespace), client.MatchingLabels{common.RayClusterOwnerKey: request.Name}); err != nil {
return reconcile.Result{}, err
}
runtimePodNameList := mapset.NewSet()
runtimePodMap := make(map[string]corev1.Pod)
for _, runtimePod := range runtimePods.Items {
runtimePodNameList.Add(runtimePod.Name)
runtimePodMap[runtimePod.Name] = runtimePod
}
log.Info("Runtime Pods", "size", len(runtimePods.Items), "runtime pods namelist", runtimePodNameList)
// record pod need to be deleted
difference := runtimePodNameList.Difference(expectedPodNameList)
// fill replicas with runtime if exists or expectedPod if not exists
var replicas []corev1.Pod
for _, pod := range expectedPods {
if runtimePodNameList.Contains(pod.Name) {
replicas = append(replicas, runtimePodMap[pod.Name])
} else {
replicas = append(replicas, pod)
}
}
// create service for head
if needServicePodMap.Cardinality() > 0 {
for elem := range needServicePodMap.Iterator().C {
podName := elem.(string)
svcConf := common.DefaultServiceConfig(*instance, podName)
rayPodSvc := common.ServiceForPod(svcConf)
if errSvc := r.Create(context.TODO(), rayPodSvc); errSvc != nil {
if errors.IsAlreadyExists(errSvc) {
log.Info("Pod service already exist,no need to create")
} else {
log.Error(errSvc, "Pod Service create error!", "Pod.Service.Error", errSvc)
return reconcile.Result{}, errSvc
}
} else {
log.Info("Pod Service create anew successfully", "podName", rayPodSvc.Name)
}
}
}
// check pod and create one by one if not exist
for i, replica := range replicas {
// create pod if not exist
if !utils.IsCreated(&replica) {
log.Info("Creating pod", "index", i, "create pod", replica.Name)
if err := r.Create(context.TODO(), &replica); err != nil {
return reconcile.Result{}, err
}
// pod created, no more work possible for this round
continue
}
}
// delete pods to desired state
if difference.Cardinality() > 0 {
log.Info("difference", "pods", difference)
for _, runtimePod := range runtimePods.Items {
if difference.Contains(runtimePod.Name) {
log.Info("Deleting pod", "namespace", runtimePod.Namespace, "name", runtimePod.Name)
if err := r.Delete(context.TODO(), &runtimePod); err != nil {
return reconcile.Result{}, err
}
if strings.EqualFold(runtimePod.Labels[common.ClusterPodType], common.Head) {
svcConf := common.DefaultServiceConfig(*instance, runtimePod.Name)
raySvcHead := common.ServiceForPod(svcConf)
log.Info("delete head service", "headName", runtimePod.Name)
if err := r.Delete(context.TODO(), raySvcHead); err != nil {
return reconcile.Result{}, err
}
}
}
}
}
return reconcile.Result{}, nil
}
// Build cluster instance pods.
func (r *RayClusterReconciler) buildPods(instance *rayiov1alpha1.RayCluster) []corev1.Pod {
var pods []corev1.Pod
if instance.Spec.Extensions != nil && len(instance.Spec.Extensions) > 0 {
for _, extension := range instance.Spec.Extensions {
var i int32 = 0
for i = 0; i < *extension.Replicas; i++ {
podType := fmt.Sprintf("%v", extension.Type)
podName := instance.Name + common.DashSymbol + extension.GroupName + common.DashSymbol + podType + common.DashSymbol + utils.FormatInt32(i)
podConf := common.DefaultPodConfig(instance, podType, podName)
podConf.Extension = extension
pod := common.BuildPod(podConf)
// Set raycluster instance as the owner and controller
if err := controllerutil.SetControllerReference(instance, pod, r.Scheme); err != nil {
log.Error(err, "Failed to set controller reference for raycluster pod")
}
pods = append(pods, *pod)
}
}
} else {
log.Info("RayCluster extensions are nil or empty")
}
return pods
}
func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&rayiov1alpha1.RayCluster{}).
Complete(r)
}
func ignoreNotFound(err error) error {
if apierrs.IsNotFound(err) {
return nil
}
return err
}
@@ -0,0 +1,65 @@
package controllers
import (
"path/filepath"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
rayv1 "ray-operator/api/v1alpha1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
// +kubebuilder:scaffold:imports
)
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
var cfg *rest.Config
var k8sClient client.Client
var testEnv *envtest.Environment
func TestAPIs(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t,
"Controller Suite",
[]Reporter{envtest.NewlineReporter{}})
}
var _ = BeforeSuite(func(done Done) {
logf.SetLogger(zap.LoggerTo(GinkgoWriter, true))
By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
}
var err error
cfg, err = testEnv.Start()
Expect(err).ToNot(HaveOccurred())
Expect(cfg).ToNot(BeNil())
err = rayv1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
// +kubebuilder:scaffold:scheme
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
Expect(err).ToNot(HaveOccurred())
Expect(k8sClient).ToNot(BeNil())
close(done)
}, 60)
var _ = AfterSuite(func() {
By("tearing down the test environment")
err := testEnv.Stop()
Expect(err).ToNot(HaveOccurred())
})
@@ -0,0 +1,28 @@
package utils
import (
corev1 "k8s.io/api/core/v1"
"strconv"
"strings"
)
// IsCreated returns true if pod has been created and is maintained by the API server
func IsCreated(pod *corev1.Pod) bool {
return pod.Status.Phase != ""
}
// Get substring before a string.
func Before(value string, a string) string {
pos := strings.Index(value, a)
if pos == -1 {
return ""
}
return value[0:pos]
}
// FormatInt returns the string representation of i in the given base,
// for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z'
// for digit values >= 10.
func FormatInt32(n int32) string {
return strconv.FormatInt(int64(n), 10)
}