feat: reconcile component dependencies

internal/controller/component_controller.go

@@ -29,7 +29,9 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/LocoStack/loco-operator/api/v1alpha1"
 	"github.com/LocoStack/loco-operator/internal/reconciler"
@@ -45,6 +47,7 @@ type ComponentReconciler struct {
 // +kubebuilder:rbac:groups=locostack.com,resources=components,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=locostack.com,resources=components/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=locostack.com,resources=components/finalizers,verbs=update
+// +kubebuilder:rbac:groups=locostack.com,resources=*,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch
@@ -99,31 +102,35 @@ func (r *ComponentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		})
 		log.Error(err, "Failed to reconcile Component", "namespace", req.NamespacedName.Namespace, "name", req.NamespacedName.Name)
 		_ = r.Status().Patch(ctx, comp, patch)
-		return ctrl.Result{}, nil
+		return ctrl.Result{}, err
 	}
 
 	comp.Status.Endpoint = fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", componentReconciler.ResourceName(""), comp.Namespace, tmpl.Spec.Runtime.Port)
 	comp.Status.ObservedGeneration = comp.Generation
-	if len(dependencies) > 0 {
-		comp.Status.Dependencies = make(map[string]map[string]int64)
-		for _, dep := range dependencies {
-			gvk, err := apiutil.GVKForObject(dep, r.Scheme)
-			if err != nil {
-				log.Error(err, "Failed to get GVK for dependency", "dependency", dep)
-				continue
-			}
-			if gvk.Group != v1alpha1.GroupVersion.Group {
-				continue
-			}
-			kind := gvk.Kind
-			name := dep.GetName()
-			generation := dep.GetGeneration()
+	// rebuilt from scratch on every reconcile, including when there are no dependencies
+	// left: dependencies that are gone have to drop out of the status too, since this is
+	// what tells them whether the Component is still serving their current generation
+	comp.Status.Dependencies = nil
+	for _, dep := range dependencies {
+		gvk, err := apiutil.GVKForObject(dep, r.Scheme)
+		if err != nil {
+			log.Error(err, "Failed to get GVK for dependency", "dependency", dep)
+			continue
+		}
+		if gvk.Group != v1alpha1.GroupVersion.Group {
+			continue
+		}
+		kind := gvk.Kind
+		name := dep.GetName()
+		generation := dep.GetGeneration()
 
-			if _, exists := comp.Status.Dependencies[kind]; !exists {
-				comp.Status.Dependencies[kind] = make(map[string]int64)
-			}
-			comp.Status.Dependencies[kind][name] = generation
+		if comp.Status.Dependencies == nil {
+			comp.Status.Dependencies = make(map[string]map[string]int64)
+		}
+		if _, exists := comp.Status.Dependencies[kind]; !exists {
+			comp.Status.Dependencies[kind] = make(map[string]int64)
 		}
+		comp.Status.Dependencies[kind][name] = generation
 	}
 	apimeta.SetStatusCondition(&comp.Status.Conditions, metav1.Condition{
 		Type:               "Reconciled",
@@ -184,13 +191,69 @@ func (r *ComponentReconciler) getComponentReconciler(comp *v1alpha1.Component, s
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *ComponentReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
+	c := mgr.GetClient()
+	builder := ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.Component{}).
 		Owns(&corev1.ConfigMap{}).
 		Owns(&corev1.Secret{}).
 		Owns(&corev1.PersistentVolumeClaim{}).
 		Owns(&corev1.Service{}).
 		Owns(&appsv1.Deployment{}).
+		// a Component renders its config from the objects it depends on, so every one of
+		// them has to be able to trigger a re-render - and with it a rollout
+		Watches(&v1alpha1.Component{}, handler.EnqueueRequestsFromMapFunc(componentDependencyMapFunc(c))).
+		Watches(&corev1.ConfigMap{}, SkillConfigMapEventHandler(c)).
+		Watches(&corev1.Secret{}, AuthSecretEventHandler(c))
+	for _, dep := range componentDependencyKinds {
+		builder = builder.Watches(dep.object, handler.EnqueueRequestsFromMapFunc(dependencyMapFunc(c, dep.kind)))
+	}
+	return builder.
 		Named("component").
 		Complete(r)
 }
+
+var componentDependencyKinds = []struct {
+	kind   string
+	object client.Object
+}{
+	{"ExternalModel", &v1alpha1.ExternalModel{}},
+	{"ManagedModel", &v1alpha1.ManagedModel{}},
+	{"ExternalTool", &v1alpha1.ExternalTool{}},
+	{"ManagedTool", &v1alpha1.ManagedTool{}},
+	{"Skill", &v1alpha1.Skill{}},
+	{"ExternalKnowledgeBase", &v1alpha1.ExternalKnowledgeBase{}},
+	{"ManagedKnowledgeBase", &v1alpha1.ManagedKnowledgeBase{}},
+	{"Agent", &v1alpha1.Agent{}},
+}
+
+func componentDependencyMapFunc(c client.Client) handler.MapFunc {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		comp, ok := obj.(*v1alpha1.Component)
+		if !ok {
+			return nil
+		}
+		stackName := ""
+		if comp.Spec.StackRef != nil {
+			stackName = comp.Spec.StackRef.Name
+		}
+		stackWide := stackName != "" && stackComponentCategories[comp.Spec.Category]
+		return dependentComponentRequests(ctx, c, comp.GetNamespace(), func(other *v1alpha1.Component) bool {
+			if other.GetName() == comp.GetName() {
+				// the Component's own events already reach the controller through For()
+				return false
+			}
+			if componentDependsOn(other, "Component", comp.GetName(), comp.GetNamespace()) {
+				return true
+			}
+			return stackWide && other.Spec.StackRef != nil && other.Spec.StackRef.Name == stackName
+		})
+	}
+}
+
+func dependencyMapFunc(c client.Client, kind string) handler.MapFunc {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		return dependentComponentRequests(ctx, c, obj.GetNamespace(), func(comp *v1alpha1.Component) bool {
+			return componentDependsOn(comp, kind, obj.GetName(), obj.GetNamespace())
+		})
+	}
+}

internal/controller/helper.go

@@ -20,11 +20,13 @@ import (
 	"context"
 
 	"github.com/LocoStack/loco-operator/api/v1alpha1"
+	"github.com/LocoStack/loco-operator/internal/reconciler"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
@@ -65,3 +67,147 @@ func dependencyRefMatches(ref corev1.ObjectReference, kind string) bool {
 	gv, err := schema.ParseGroupVersion(ref.APIVersion)
 	return err == nil && gv.Group == v1alpha1.GroupVersion.Group
 }
+
+// stackComponentCategories are the Component categories that the rest of a Stack resolves
+// implicitly, through the Stack rather than through spec.dependencies: the gateway URL and
+// master key, the vector and graph store URLs and the tracing collector endpoint are all
+// rendered into the configs of the other Components, so a change to one of these has to
+// re-render the rest of the Stack.
+var stackComponentCategories = map[string]bool{
+	"Gateway":       true,
+	"VectorStore":   true,
+	"GraphStore":    true,
+	"Observability": true,
+}
+
+// SkillConfigMapEventHandler maps a Skill's ConfigMap to the Components that mount it. The
+// ConfigMap is owned by the Skill rather than by the Component, so watching the Skill alone
+// would race its ConfigMap: a Component reconciled off the Skill event can still read the
+// previous file set, and no further event would arrive to correct it.
+func SkillConfigMapEventHandler(c client.Client) handler.EventHandler {
+	return handler.EnqueueRequestsFromMapFunc(skillConfigMapMapFunc(c))
+}
+
+func skillConfigMapMapFunc(c client.Client) handler.MapFunc {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		if _, ok := obj.GetAnnotations()[v1alpha1.SkillFilePathsAnnotation]; !ok {
+			return nil
+		}
+		return dependentComponentRequests(ctx, c, obj.GetNamespace(), func(comp *v1alpha1.Component) bool {
+			return componentDependsOn(comp, "Skill", obj.GetName(), obj.GetNamespace())
+		})
+	}
+}
+
+// AuthSecretEventHandler maps a Secret to the Components whose config embeds a credential
+// read from it. Model and tool credentials reach a Pod as env vars sourced from these
+// Secrets, and a Pod's environment is fixed for its lifetime, so a rotated key only takes
+// effect once the Pod is replaced - which needs the Components depending on that model or
+// tool to re-render and roll out.
+func AuthSecretEventHandler(c client.Client) handler.EventHandler {
+	return handler.EnqueueRequestsFromMapFunc(authSecretMapFunc(c))
+}
+
+func authSecretMapFunc(c client.Client) handler.MapFunc {
+	return func(ctx context.Context, obj client.Object) []reconcile.Request {
+		log := logf.FromContext(ctx)
+		namespace := obj.GetNamespace()
+		// kind -> names of the dependencies reading a credential from this Secret
+		refs := make(map[string]map[string]bool)
+		addRef := func(kind, name string, auth *v1alpha1.AuthSpec) {
+			if !authReadsSecret(auth, obj.GetName()) {
+				return
+			}
+			if refs[kind] == nil {
+				refs[kind] = make(map[string]bool)
+			}
+			refs[kind][name] = true
+		}
+
+		emList := &v1alpha1.ExternalModelList{}
+		if err := c.List(ctx, emList, client.InNamespace(namespace)); err != nil {
+			log.Error(err, "Failed to list ExternalModels for Secret event", "namespace", namespace, "secret", obj.GetName())
+			return nil
+		}
+		for i := range emList.Items {
+			addRef("ExternalModel", emList.Items[i].Name, emList.Items[i].Spec.Auth)
+		}
+		etList := &v1alpha1.ExternalToolList{}
+		if err := c.List(ctx, etList, client.InNamespace(namespace)); err != nil {
+			log.Error(err, "Failed to list ExternalTools for Secret event", "namespace", namespace, "secret", obj.GetName())
+			return nil
+		}
+		for i := range etList.Items {
+			addRef("ExternalTool", etList.Items[i].Name, etList.Items[i].Spec.Auth)
+		}
+		mtList := &v1alpha1.ManagedToolList{}
+		if err := c.List(ctx, mtList, client.InNamespace(namespace)); err != nil {
+			log.Error(err, "Failed to list ManagedTools for Secret event", "namespace", namespace, "secret", obj.GetName())
+			return nil
+		}
+		for i := range mtList.Items {
+			addRef("ManagedTool", mtList.Items[i].Name, mtList.Items[i].Spec.Auth)
+		}
+
+		if len(refs) == 0 {
+			return nil
+		}
+		return dependentComponentRequests(ctx, c, namespace, func(comp *v1alpha1.Component) bool {
+			for kind, names := range refs {
+				for name := range names {
+					if componentDependsOn(comp, kind, name, namespace) {
+						return true
+					}
+				}
+			}
+			return false
+		})
+	}
+}
+
+// dependentComponentRequests enqueues every Component in the namespace that match reports
+// as depending on the object that triggered the event.
+func dependentComponentRequests(ctx context.Context, c client.Client, namespace string, match func(*v1alpha1.Component) bool) []reconcile.Request {
+	log := logf.FromContext(ctx)
+	compList := &v1alpha1.ComponentList{}
+	if err := c.List(ctx, compList, client.InNamespace(namespace)); err != nil {
+		log.Error(err, "Failed to list Components for dependency event", "namespace", namespace)
+		return nil
+	}
+	var reqs []reconcile.Request
+	for i := range compList.Items {
+		comp := &compList.Items[i]
+		if !match(comp) {
+			continue
+		}
+		reqs = append(reqs, reconcile.Request{
+			NamespacedName: types.NamespacedName{Name: comp.GetName(), Namespace: comp.GetNamespace()},
+		})
+	}
+	return reqs
+}
+
+// componentDependsOn reports whether comp declares the named object as a dependency. A
+// dependency reference with no namespace means the Component's own namespace, which is the
+// only namespace this is ever asked about.
+func componentDependsOn(comp *v1alpha1.Component, kind, name, namespace string) bool {
+	for _, dep := range comp.Spec.Dependencies {
+		if dep.Name != name || !dependencyRefMatches(dep, kind) {
+			continue
+		}
+		if dep.Namespace == "" || dep.Namespace == namespace {
+			return true
+		}
+	}
+	return false
+}
+
+// authReadsSecret reports whether auth sources any of its credentials from the named Secret.
+func authReadsSecret(auth *v1alpha1.AuthSpec, name string) bool {
+	for _, ref := range reconciler.AuthSecretRefs(auth) {
+		if ref.Name == name {
+			return true
+		}
+	}
+	return false
+}