cyrus 8 месяцев назад
Родитель
Сommit
4cb88cb06f

+ 3 - 1
k8s-orchestrator/cmd/orchestrator-controller/main.go

@@ -8,6 +8,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -38,6 +39,7 @@ func main() {
 	scheme := runtime.NewScheme()
 	utilruntime.Must(appsv1.AddToScheme(scheme))
 	utilruntime.Must(corev1.AddToScheme(scheme))
+	utilruntime.Must(rbacv1.AddToScheme(scheme))
 	utilruntime.Must(orcv1.AddToScheme(scheme))
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -55,7 +57,7 @@ func main() {
 
 	onboarder := &onboarding.SSHOnboarder{
 		DialTimeout:    15 * time.Second,
-		CommandTimeout: 8 * time.Minute,
+		CommandTimeout: 15 * time.Minute,
 		DefaultJoinCmd: kubeadmJoinCmd,
 	}
 

+ 8 - 4
k8s-orchestrator/internal/controller/clusteroperation_controller.go

@@ -656,10 +656,14 @@ func (r *ClusterOperationReconciler) ensureNoRunningPodsOnNodesInNamespace(ctx c
 		if pod.Spec.NodeName == "" {
 			continue
 		}
-		if pod.Labels != nil &&
-			pod.Labels[util.LabelManagedBy] == util.ManagedByValue &&
-			pod.Labels[util.LabelComponent] == util.ComponentSparkPrePull {
-			continue
+		if pod.Labels != nil && pod.Labels[util.LabelManagedBy] == util.ManagedByValue {
+			// Skip managed infrastructure pods: prepull DaemonSets and StarRocks FE/BE pods.
+			if pod.Labels[util.LabelComponent] == util.ComponentSparkPrePull {
+				continue
+			}
+			if _, hasStarRocksComponent := pod.Labels[util.LabelStarRocksComponent]; hasStarRocksComponent {
+				continue
+			}
 		}
 		if _, ok := nodeSet[pod.Spec.NodeName]; !ok {
 			if _, ok := k8sNames[pod.Spec.NodeName]; !ok {

+ 29 - 5
k8s-orchestrator/internal/controller/nodeinventory_controller.go

@@ -67,15 +67,24 @@ func (r *NodeInventoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 			return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
 		}
 
-		nodeName, err := k8s.FindNodeByInternalIP(ctx, r.Client, nodeInv.Spec.Address)
-		if err != nil {
-			if apierrors.IsNotFound(err) {
-				return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+		// Prefer the name already resolved by ensureNodeJoined (set via --node-name).
+		// Fall back to IP-based lookup for nodes joined without an explicit name.
+		nodeName := nodeInv.Status.KubernetesNodeName
+		if nodeName == "" {
+			name, err := k8s.FindNodeByInternalIP(ctx, r.Client, nodeInv.Spec.Address)
+			if err != nil {
+				if apierrors.IsNotFound(err) {
+					return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+				}
+				return ctrl.Result{}, err
 			}
-			return ctrl.Result{}, err
+			nodeName = name
 		}
 		var k8sNode corev1.Node
 		if err := r.Get(ctx, types.NamespacedName{Name: nodeName}, &k8sNode); err != nil {
+			if apierrors.IsNotFound(err) {
+				return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
+			}
 			return ctrl.Result{}, err
 		}
 		nodeInv.Status.KubernetesNodeName = nodeName
@@ -112,6 +121,21 @@ func (r *NodeInventoryReconciler) ensureNodeJoined(ctx context.Context, nodeInv
 	if nodeInv.Status.KubernetesNodeName != "" {
 		return nil
 	}
+
+	// Check by predictable node name (set via --node-name in kubeadm join).
+	// This handles the case where join succeeded but the node name wasn't yet persisted.
+	var candidate corev1.Node
+	if err := r.Get(ctx, types.NamespacedName{Name: nodeInv.Name}, &candidate); err == nil {
+		nodeInv.Status.KubernetesNodeName = candidate.Name
+		return nil
+	}
+
+	// Fallback: search by internal IP (covers nodes joined without --node-name).
+	if existing, err := k8s.FindNodeByInternalIP(ctx, r.Client, nodeInv.Spec.Address); err == nil && existing != "" {
+		nodeInv.Status.KubernetesNodeName = existing
+		return nil
+	}
+
 	if r.Onboarder == nil {
 		return fmt.Errorf("onboarder is not configured")
 	}

+ 3 - 0
k8s-orchestrator/internal/controller/sparkvirtualcluster_controller.go

@@ -99,6 +99,9 @@ func (r *SparkVirtualClusterReconciler) Reconcile(ctx context.Context, req ctrl.
 	if err := k8s.EnsureNamespace(ctx, r.Client, ns); err != nil {
 		return ctrl.Result{}, err
 	}
+	if err := k8s.EnsureSparkOperatorRBAC(ctx, r.Client, ns); err != nil {
+		return ctrl.Result{}, err
+	}
 
 	targetNodes := normalizeNodeList(cluster.Spec.NodeRefs)
 	previousNodes := normalizeNodeList(cluster.Status.AssignedNodes)

+ 11 - 2
k8s-orchestrator/internal/controller/starrocksvirtualcluster_controller.go

@@ -351,10 +351,19 @@ func deriveMinResources(nodes []orcv1.NodeInventory) (*orcv1.StarRocksResourceSp
 	if minCPU <= 0 || minRAM <= 0 {
 		return nil, fmt.Errorf("invalid node capacity cpu=%d ram=%d", minCPU, minRAM)
 	}
+	// Use 75% of CPU/RAM for requests to leave headroom for OS/system processes.
+	reqCPU := minCPU * 3 / 4
+	if reqCPU < 1 {
+		reqCPU = 1
+	}
+	reqRAM := minRAM * 3 / 4
+	if reqRAM < 1 {
+		reqRAM = 1
+	}
 	return &orcv1.StarRocksResourceSpec{
 		Requests: orcv1.StarRocksResourcePair{
-			CPU:    strconv.Itoa(int(minCPU)),
-			Memory: fmt.Sprintf("%dGi", minRAM),
+			CPU:    strconv.Itoa(int(reqCPU)),
+			Memory: fmt.Sprintf("%dGi", reqRAM),
 		},
 		Limits: orcv1.StarRocksResourcePair{
 			CPU:    strconv.Itoa(int(minCPU)),

+ 116 - 5
k8s-orchestrator/internal/k8s/node.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -39,13 +40,30 @@ func FindNodeByInternalIP(ctx context.Context, c ctrlclient.Client, ip string) (
 }
 
 func EnsureNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName, clusterID string) error {
-	return ensureNodeIsolationWithKeys(ctx, c, nodeName, clusterID, util.LabelSparkClusterID, util.TaintKeySparkCluster)
+	// Spark isolation uses label+nodeSelector only (no taint) because Spark Operator v2
+	// doesn't reliably apply driver.tolerations to the driver pod.
+	// The nodeSelector `spark.cluster.id=clusterID` is injected via SparkConf and ensures
+	// Spark pods run on cluster nodes; a NoSchedule taint would block them.
+	return ensureLabelOnly(ctx, c, nodeName, clusterID, util.LabelSparkClusterID)
 }
 
 func EnsureStarRocksNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName, clusterID string) error {
 	return ensureNodeIsolationWithKeys(ctx, c, nodeName, clusterID, util.LabelStarRocksClusterID, util.TaintKeyStarRocksCluster)
 }
 
+func ensureLabelOnly(ctx context.Context, c ctrlclient.Client, nodeName, clusterID, labelKey string) error {
+	var node corev1.Node
+	if err := c.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil {
+		return err
+	}
+	if node.Labels == nil {
+		node.Labels = map[string]string{}
+	}
+	node.Labels[labelKey] = clusterID
+	node.Labels[util.LabelManagedBy] = util.ManagedByValue
+	return c.Update(ctx, &node)
+}
+
 func ensureNodeIsolationWithKeys(ctx context.Context, c ctrlclient.Client, nodeName, clusterID, labelKey, taintKey string) error {
 	var node corev1.Node
 	if err := c.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil {
@@ -57,15 +75,18 @@ func ensureNodeIsolationWithKeys(ctx context.Context, c ctrlclient.Client, nodeN
 	node.Labels[labelKey] = clusterID
 	node.Labels[util.LabelManagedBy] = util.ManagedByValue
 
-	taint := corev1.Taint{Key: taintKey, Value: clusterID, Effect: corev1.TaintEffectNoSchedule}
-	if !containsTaint(node.Spec.Taints, taint) {
-		node.Spec.Taints = append(node.Spec.Taints, taint)
+	if taintKey != "" {
+		taint := corev1.Taint{Key: taintKey, Value: clusterID, Effect: corev1.TaintEffectNoSchedule}
+		if !containsTaint(node.Spec.Taints, taint) {
+			node.Spec.Taints = append(node.Spec.Taints, taint)
+		}
 	}
 	return c.Update(ctx, &node)
 }
 
 func ClearNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName string) error {
-	return clearNodeIsolationWithKeys(ctx, c, nodeName, util.LabelSparkClusterID, util.TaintKeySparkCluster)
+	// Spark uses label-only isolation (no taint), so we only need to clear the label.
+	return clearNodeIsolationWithKeys(ctx, c, nodeName, util.LabelSparkClusterID, "")
 }
 
 func ClearStarRocksNodeIsolation(ctx context.Context, c ctrlclient.Client, nodeName string) error {
@@ -111,6 +132,96 @@ func EnsureNamespace(ctx context.Context, c ctrlclient.Client, namespace string)
 	return c.Create(ctx, &newNS)
 }
 
+// EnsureSparkOperatorRBAC creates the Role and RoleBinding needed for the
+// Spark operator controller and Spark driver pods in the given namespace.
+// This must be called whenever a new SparkVirtualCluster namespace is created.
+func EnsureSparkOperatorRBAC(ctx context.Context, c ctrlclient.Client, namespace string) error {
+	// Role for spark-operator-controller to manage SparkApplications and pods in this namespace.
+	controllerRole := &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "spark-operator-controller",
+			Namespace: namespace,
+			Labels:    map[string]string{util.LabelManagedBy: util.ManagedByValue},
+		},
+		Rules: []rbacv1.PolicyRule{
+			{APIGroups: []string{""}, Resources: []string{"pods", "configmaps", "persistentvolumeclaims", "services"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"}},
+			{APIGroups: []string{""}, Resources: []string{"events"}, Verbs: []string{"create", "update", "patch"}},
+			{APIGroups: []string{"sparkoperator.k8s.io"}, Resources: []string{"sparkapplications", "scheduledsparkapplications", "sparkconnects"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}},
+			{APIGroups: []string{"sparkoperator.k8s.io"}, Resources: []string{"sparkapplications/status", "sparkapplications/finalizers", "scheduledsparkapplications/status", "scheduledsparkapplications/finalizers", "sparkconnects/status", "sparkconnects/finalizers"}, Verbs: []string{"get", "update", "patch"}},
+		},
+	}
+	if err := applyRole(ctx, c, controllerRole); err != nil {
+		return fmt.Errorf("ensure spark-operator-controller role: %w", err)
+	}
+
+	controllerRB := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "spark-operator-controller",
+			Namespace: namespace,
+			Labels:    map[string]string{util.LabelManagedBy: util.ManagedByValue},
+		},
+		RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "Role", Name: "spark-operator-controller"},
+		Subjects: []rbacv1.Subject{
+			{Kind: "ServiceAccount", Name: "spark-operator-controller", Namespace: "spark-operator"},
+		},
+	}
+	if err := applyRoleBinding(ctx, c, controllerRB); err != nil {
+		return fmt.Errorf("ensure spark-operator-controller rolebinding: %w", err)
+	}
+
+	// Role for Spark driver pods (running as the default ServiceAccount) to manage executor pods.
+	driverRole := &rbacv1.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "spark-operator-spark",
+			Namespace: namespace,
+			Labels:    map[string]string{util.LabelManagedBy: util.ManagedByValue},
+		},
+		Rules: []rbacv1.PolicyRule{
+			{APIGroups: []string{""}, Resources: []string{"pods", "configmaps", "persistentvolumeclaims", "services"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"}},
+		},
+	}
+	if err := applyRole(ctx, c, driverRole); err != nil {
+		return fmt.Errorf("ensure spark-operator-spark role: %w", err)
+	}
+
+	driverRB := &rbacv1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "spark-operator-spark",
+			Namespace: namespace,
+			Labels:    map[string]string{util.LabelManagedBy: util.ManagedByValue},
+		},
+		RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "Role", Name: "spark-operator-spark"},
+		Subjects: []rbacv1.Subject{
+			{Kind: "ServiceAccount", Name: "default", Namespace: namespace},
+		},
+	}
+	return applyRoleBinding(ctx, c, driverRB)
+}
+
+func applyRole(ctx context.Context, c ctrlclient.Client, role *rbacv1.Role) error {
+	var existing rbacv1.Role
+	err := c.Get(ctx, types.NamespacedName{Name: role.Name, Namespace: role.Namespace}, &existing)
+	if err == nil {
+		return nil
+	}
+	if !apierrors.IsNotFound(err) {
+		return err
+	}
+	return c.Create(ctx, role)
+}
+
+func applyRoleBinding(ctx context.Context, c ctrlclient.Client, rb *rbacv1.RoleBinding) error {
+	var existing rbacv1.RoleBinding
+	err := c.Get(ctx, types.NamespacedName{Name: rb.Name, Namespace: rb.Namespace}, &existing)
+	if err == nil {
+		return nil
+	}
+	if !apierrors.IsNotFound(err) {
+		return err
+	}
+	return c.Create(ctx, rb)
+}
+
 func containsTaint(existing []corev1.Taint, target corev1.Taint) bool {
 	for _, t := range existing {
 		if t.Key == target.Key && t.Value == target.Value && t.Effect == target.Effect {

+ 18 - 23
k8s-orchestrator/internal/k8s/spark.go

@@ -99,6 +99,7 @@ func injectIsolation(spec map[string]interface{}, clusterID string) {
 	if spec == nil {
 		return
 	}
+	// Set nodeSelector on driver and executor specs (for webhook-applied node selection).
 	for _, role := range []string{"driver", "executor"} {
 		roleMap, ok := toMap(spec[role])
 		if !ok {
@@ -111,18 +112,25 @@ func injectIsolation(spec map[string]interface{}, clusterID string) {
 		}
 		nodeSelector[util.LabelClusterID] = clusterID
 		roleMap["nodeSelector"] = nodeSelector
+	}
 
-		tolerations := toSlice(roleMap["tolerations"])
-		if !hasSparkClusterToleration(tolerations, clusterID) {
-			tolerations = append(tolerations, map[string]interface{}{
-				"key":      util.TaintKeySparkCluster,
-				"operator": "Equal",
-				"value":    clusterID,
-				"effect":   "NoSchedule",
-			})
-		}
-		roleMap["tolerations"] = tolerations
+	// Also inject via SparkConf for Spark-on-K8s native node selector support.
+	// Spark Operator v2 reads spark.kubernetes.{driver,executor}.node.selector.* and applies
+	// the nodeSelector to the pod directly, which is more reliable than spec.driver.nodeSelector.
+	sparkConf, ok := toMap(spec["sparkConf"])
+	if !ok {
+		sparkConf = map[string]interface{}{}
+	}
+	labelKey := util.LabelClusterID
+	driverKey := "spark.kubernetes.driver.node.selector." + labelKey
+	execKey := "spark.kubernetes.executor.node.selector." + labelKey
+	if _, exists := sparkConf[driverKey]; !exists {
+		sparkConf[driverKey] = clusterID
 	}
+	if _, exists := sparkConf[execKey]; !exists {
+		sparkConf[execKey] = clusterID
+	}
+	spec["sparkConf"] = sparkConf
 }
 
 func toMap(v interface{}) (map[string]interface{}, bool) {
@@ -145,19 +153,6 @@ func toSlice(v interface{}) []interface{} {
 	return []interface{}{}
 }
 
-func hasSparkClusterToleration(tolerations []interface{}, clusterID string) bool {
-	for _, t := range tolerations {
-		m, ok := t.(map[string]interface{})
-		if !ok {
-			continue
-		}
-		if m["key"] == util.TaintKeySparkCluster && m["value"] == clusterID {
-			return true
-		}
-	}
-	return false
-}
-
 // DeleteSparkApplication deletes a SparkApplication by name and namespace
 func DeleteSparkApplication(ctx context.Context, c ctrlclient.Client, namespace, name string) error {
 	obj := &unstructured.Unstructured{

+ 82 - 10
k8s-orchestrator/internal/onboarding/onboarder.go

@@ -28,6 +28,61 @@ type SSHOnboarder struct {
 	DefaultJoinCmd string
 }
 
+// k8sSetupScript installs all prerequisites for kubeadm join on a fresh Ubuntu 22.04 node.
+// The script is idempotent: if kubeadm + kubelet + conntrack are already present it exits immediately.
+const k8sSetupScript = `#!/bin/bash
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+# Idempotency check: skip if everything already installed
+if command -v kubeadm >/dev/null 2>&1 && \
+   command -v kubelet >/dev/null 2>&1 && \
+   command -v conntrack >/dev/null 2>&1; then
+    echo "[setup] prerequisites already installed, skipping"
+    exit 0
+fi
+
+echo "[setup] step 1/4: disable swap"
+swapoff -a
+sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
+
+echo "[setup] step 2/4: kernel modules & sysctl"
+cat > /etc/modules-load.d/k8s.conf <<EOF
+overlay
+br_netfilter
+EOF
+modprobe overlay
+modprobe br_netfilter
+cat > /etc/sysctl.d/k8s.conf <<EOF
+net.bridge.bridge-nf-call-iptables  = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward                 = 1
+EOF
+sysctl --system
+
+echo "[setup] step 3/4: install containerd"
+apt-get update -qq
+apt-get install -y -qq containerd apt-transport-https ca-certificates curl gpg conntrack socat
+mkdir -p /etc/containerd
+containerd config default > /etc/containerd/config.toml
+sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
+systemctl restart containerd
+systemctl enable containerd
+
+echo "[setup] step 4/4: install kubeadm/kubelet/kubectl v1.31"
+mkdir -p /etc/apt/keyrings
+curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key \
+  | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' \
+  > /etc/apt/sources.list.d/kubernetes.list
+apt-get update -qq
+apt-get install -y -qq kubelet kubeadm kubectl
+apt-mark hold kubelet kubeadm kubectl
+systemctl enable --now kubelet
+
+echo "[setup] done"
+`
+
 func (s *SSHOnboarder) Onboard(ctx context.Context, req NodeJoinRequest) error {
 	if req.Address == "" {
 		return fmt.Errorf("address is required")
@@ -46,22 +101,22 @@ func (s *SSHOnboarder) Onboard(ctx context.Context, req NodeJoinRequest) error {
 		return fmt.Errorf("join command is empty")
 	}
 
-	timeout := s.DialTimeout
-	if timeout == 0 {
-		timeout = 15 * time.Second
+	dialTimeout := s.DialTimeout
+	if dialTimeout == 0 {
+		dialTimeout = 15 * time.Second
 	}
-	config := &ssh.ClientConfig{
+	sshCfg := &ssh.ClientConfig{
 		User:            req.User,
 		Auth:            []ssh.AuthMethod{ssh.Password(req.Password)},
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec
-		Timeout:         timeout,
+		Timeout:         dialTimeout,
 	}
 
-	conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", req.Address, req.Port), timeout)
+	conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", req.Address, req.Port), dialTimeout)
 	if err != nil {
 		return fmt.Errorf("dial node: %w", err)
 	}
-	clientConn, chans, reqs, err := ssh.NewClientConn(conn, fmt.Sprintf("%s:%d", req.Address, req.Port), config)
+	clientConn, chans, reqs, err := ssh.NewClientConn(conn, fmt.Sprintf("%s:%d", req.Address, req.Port), sshCfg)
 	if err != nil {
 		_ = conn.Close()
 		return fmt.Errorf("ssh handshake: %w", err)
@@ -69,6 +124,24 @@ func (s *SSHOnboarder) Onboard(ctx context.Context, req NodeJoinRequest) error {
 	client := ssh.NewClient(clientConn, chans, reqs)
 	defer client.Close()
 
+	// Step 1: install prerequisites (idempotent)
+	if err := s.runRemoteCmd(ctx, client, k8sSetupScript); err != nil {
+		return fmt.Errorf("node setup: %w", err)
+	}
+
+	// Step 2: kubeadm join
+	// Append --node-name so the K8s node name is predictable and matches NodeInventory name.
+	joinWithName := join
+	if req.NodeNameHint != "" {
+		joinWithName = fmt.Sprintf("%s --node-name %s", join, req.NodeNameHint)
+	}
+	if err := s.runRemoteCmd(ctx, client, fmt.Sprintf("set -euo pipefail; %s", joinWithName)); err != nil {
+		return fmt.Errorf("run join command: %w", err)
+	}
+	return nil
+}
+
+func (s *SSHOnboarder) runRemoteCmd(ctx context.Context, client *ssh.Client, cmd string) error {
 	sess, err := client.NewSession()
 	if err != nil {
 		return fmt.Errorf("new ssh session: %w", err)
@@ -82,7 +155,6 @@ func (s *SSHOnboarder) Onboard(ctx context.Context, req NodeJoinRequest) error {
 		defer cancel()
 	}
 
-	cmd := fmt.Sprintf("set -euo pipefail; %s", join)
 	done := make(chan error, 1)
 	go func() {
 		done <- sess.Run(cmd)
@@ -91,11 +163,11 @@ func (s *SSHOnboarder) Onboard(ctx context.Context, req NodeJoinRequest) error {
 	select {
 	case err := <-done:
 		if err != nil {
-			return fmt.Errorf("run join command: %w", err)
+			return err
 		}
 		return nil
 	case <-runCtx.Done():
 		_ = sess.Close()
-		return fmt.Errorf("run join command timed out: %w", runCtx.Err())
+		return fmt.Errorf("timed out: %w", runCtx.Err())
 	}
 }