|
|
@@ -5,6 +5,7 @@ import (
|
|
|
"fmt"
|
|
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
|
+ rbacv1 "k8s.io/api/rbac/v1"
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
|
"k8s.io/apimachinery/pkg/types"
|
|
|
@@ -39,13 +40,30 @@ func FindNodeByInternalIP(ctx context.Context, c ctrlclient.Client, ip string) (
|
|
|
}
|
|
|
|
|
|
func EnsureNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName, clusterID string) error {
|
|
|
- return ensureNodeIsolationWithKeys(ctx, c, nodeName, clusterID, util.LabelSparkClusterID, util.TaintKeySparkCluster)
|
|
|
+ // Spark isolation uses label+nodeSelector only (no taint) because Spark Operator v2
|
|
|
+ // doesn't reliably apply driver.tolerations to the driver pod.
|
|
|
+ // The nodeSelector `spark.cluster.id=clusterID` is injected via SparkConf and ensures
|
|
|
+ // Spark pods run on cluster nodes; a NoSchedule taint would block them.
|
|
|
+ return ensureLabelOnly(ctx, c, nodeName, clusterID, util.LabelSparkClusterID)
|
|
|
}
|
|
|
|
|
|
func EnsureStarRocksNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName, clusterID string) error {
|
|
|
return ensureNodeIsolationWithKeys(ctx, c, nodeName, clusterID, util.LabelStarRocksClusterID, util.TaintKeyStarRocksCluster)
|
|
|
}
|
|
|
|
|
|
+func ensureLabelOnly(ctx context.Context, c ctrlclient.Client, nodeName, clusterID, labelKey string) error {
|
|
|
+ var node corev1.Node
|
|
|
+ if err := c.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ if node.Labels == nil {
|
|
|
+ node.Labels = map[string]string{}
|
|
|
+ }
|
|
|
+ node.Labels[labelKey] = clusterID
|
|
|
+ node.Labels[util.LabelManagedBy] = util.ManagedByValue
|
|
|
+ return c.Update(ctx, &node)
|
|
|
+}
|
|
|
+
|
|
|
func ensureNodeIsolationWithKeys(ctx context.Context, c ctrlclient.Client, nodeName, clusterID, labelKey, taintKey string) error {
|
|
|
var node corev1.Node
|
|
|
if err := c.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil {
|
|
|
@@ -57,15 +75,18 @@ func ensureNodeIsolationWithKeys(ctx context.Context, c ctrlclient.Client, nodeN
|
|
|
node.Labels[labelKey] = clusterID
|
|
|
node.Labels[util.LabelManagedBy] = util.ManagedByValue
|
|
|
|
|
|
- taint := corev1.Taint{Key: taintKey, Value: clusterID, Effect: corev1.TaintEffectNoSchedule}
|
|
|
- if !containsTaint(node.Spec.Taints, taint) {
|
|
|
- node.Spec.Taints = append(node.Spec.Taints, taint)
|
|
|
+ if taintKey != "" {
|
|
|
+ taint := corev1.Taint{Key: taintKey, Value: clusterID, Effect: corev1.TaintEffectNoSchedule}
|
|
|
+ if !containsTaint(node.Spec.Taints, taint) {
|
|
|
+ node.Spec.Taints = append(node.Spec.Taints, taint)
|
|
|
+ }
|
|
|
}
|
|
|
return c.Update(ctx, &node)
|
|
|
}
|
|
|
|
|
|
func ClearNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName string) error {
|
|
|
- return clearNodeIsolationWithKeys(ctx, c, nodeName, util.LabelSparkClusterID, util.TaintKeySparkCluster)
|
|
|
+ // Spark uses label-only isolation (no taint), so we only need to clear the label.
|
|
|
+ return clearNodeIsolationWithKeys(ctx, c, nodeName, util.LabelSparkClusterID, "")
|
|
|
}
|
|
|
|
|
|
func ClearStarRocksNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName string) error {
|
|
|
@@ -111,6 +132,96 @@ func EnsureNamespace(ctx context.Context, c ctrlclient.Client, namespace string)
|
|
|
return c.Create(ctx, &newNS)
|
|
|
}
|
|
|
|
|
|
+// EnsureSparkOperatorRBAC creates the Role and RoleBinding needed for the
|
|
|
+// Spark operator controller and Spark driver pods in the given namespace.
|
|
|
+// This must be called whenever a new SparkVirtualCluster namespace is created.
|
|
|
+func EnsureSparkOperatorRBAC(ctx context.Context, c ctrlclient.Client, namespace string) error {
|
|
|
+ // Role for spark-operator-controller to manage SparkApplications and pods in this namespace.
|
|
|
+ controllerRole := &rbacv1.Role{
|
|
|
+ ObjectMeta: metav1.ObjectMeta{
|
|
|
+ Name: "spark-operator-controller",
|
|
|
+ Namespace: namespace,
|
|
|
+ Labels: map[string]string{util.LabelManagedBy: util.ManagedByValue},
|
|
|
+ },
|
|
|
+ Rules: []rbacv1.PolicyRule{
|
|
|
+ {APIGroups: []string{""}, Resources: []string{"pods", "configmaps", "persistentvolumeclaims", "services"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"}},
|
|
|
+ {APIGroups: []string{""}, Resources: []string{"events"}, Verbs: []string{"create", "update", "patch"}},
|
|
|
+ {APIGroups: []string{"sparkoperator.k8s.io"}, Resources: []string{"sparkapplications", "scheduledsparkapplications", "sparkconnects"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}},
|
|
|
+ {APIGroups: []string{"sparkoperator.k8s.io"}, Resources: []string{"sparkapplications/status", "sparkapplications/finalizers", "scheduledsparkapplications/status", "scheduledsparkapplications/finalizers", "sparkconnects/status", "sparkconnects/finalizers"}, Verbs: []string{"get", "update", "patch"}},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ if err := applyRole(ctx, c, controllerRole); err != nil {
|
|
|
+ return fmt.Errorf("ensure spark-operator-controller role: %w", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ controllerRB := &rbacv1.RoleBinding{
|
|
|
+ ObjectMeta: metav1.ObjectMeta{
|
|
|
+ Name: "spark-operator-controller",
|
|
|
+ Namespace: namespace,
|
|
|
+ Labels: map[string]string{util.LabelManagedBy: util.ManagedByValue},
|
|
|
+ },
|
|
|
+ RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "Role", Name: "spark-operator-controller"},
|
|
|
+ Subjects: []rbacv1.Subject{
|
|
|
+ {Kind: "ServiceAccount", Name: "spark-operator-controller", Namespace: "spark-operator"},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ if err := applyRoleBinding(ctx, c, controllerRB); err != nil {
|
|
|
+ return fmt.Errorf("ensure spark-operator-controller rolebinding: %w", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ // Role for Spark driver pods (running as the default ServiceAccount) to manage executor pods.
|
|
|
+ driverRole := &rbacv1.Role{
|
|
|
+ ObjectMeta: metav1.ObjectMeta{
|
|
|
+ Name: "spark-operator-spark",
|
|
|
+ Namespace: namespace,
|
|
|
+ Labels: map[string]string{util.LabelManagedBy: util.ManagedByValue},
|
|
|
+ },
|
|
|
+ Rules: []rbacv1.PolicyRule{
|
|
|
+ {APIGroups: []string{""}, Resources: []string{"pods", "configmaps", "persistentvolumeclaims", "services"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"}},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ if err := applyRole(ctx, c, driverRole); err != nil {
|
|
|
+ return fmt.Errorf("ensure spark-operator-spark role: %w", err)
|
|
|
+ }
|
|
|
+
|
|
|
+ driverRB := &rbacv1.RoleBinding{
|
|
|
+ ObjectMeta: metav1.ObjectMeta{
|
|
|
+ Name: "spark-operator-spark",
|
|
|
+ Namespace: namespace,
|
|
|
+ Labels: map[string]string{util.LabelManagedBy: util.ManagedByValue},
|
|
|
+ },
|
|
|
+ RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "Role", Name: "spark-operator-spark"},
|
|
|
+ Subjects: []rbacv1.Subject{
|
|
|
+ {Kind: "ServiceAccount", Name: "default", Namespace: namespace},
|
|
|
+ },
|
|
|
+ }
|
|
|
+ return applyRoleBinding(ctx, c, driverRB)
|
|
|
+}
|
|
|
+
|
|
|
+func applyRole(ctx context.Context, c ctrlclient.Client, role *rbacv1.Role) error {
|
|
|
+ var existing rbacv1.Role
|
|
|
+ err := c.Get(ctx, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, &existing)
|
|
|
+ if err == nil {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+ if !apierrors.IsNotFound(err) {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ return c.Create(ctx, role)
|
|
|
+}
|
|
|
+
|
|
|
+func applyRoleBinding(ctx context.Context, c ctrlclient.Client, rb *rbacv1.RoleBinding) error {
|
|
|
+ var existing rbacv1.RoleBinding
|
|
|
+ err := c.Get(ctx, types.NamespacedName{Name: rb.Name, Namespace: rb.Namespace}, &existing)
|
|
|
+ if err == nil {
|
|
|
+ return nil
|
|
|
+ }
|
|
|
+ if !apierrors.IsNotFound(err) {
|
|
|
+ return err
|
|
|
+ }
|
|
|
+ return c.Create(ctx, rb)
|
|
|
+}
|
|
|
+
|
|
|
func containsTaint(existing []corev1.Taint, target corev1.Taint) bool {
|
|
|
for _, t := range existing {
|
|
|
if t.Key == target.Key && t.Value == target.Value && t.Effect == target.Effect {
|