stack_controller.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419
  1. /*
  2. Copyright 2026 LocoStack.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package controller
  14. import (
  15. "context"
  16. "fmt"
  17. "maps"
  18. "strings"
  19. corev1 "k8s.io/api/core/v1"
  20. "k8s.io/apimachinery/pkg/api/equality"
  21. apierrors "k8s.io/apimachinery/pkg/api/errors"
  22. apimeta "k8s.io/apimachinery/pkg/api/meta"
  23. "k8s.io/apimachinery/pkg/api/resource"
  24. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  25. "k8s.io/apimachinery/pkg/runtime"
  26. "k8s.io/apimachinery/pkg/types"
  27. "k8s.io/utils/ptr"
  28. ctrl "sigs.k8s.io/controller-runtime"
  29. "sigs.k8s.io/controller-runtime/pkg/client"
  30. "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
  31. "sigs.k8s.io/controller-runtime/pkg/handler"
  32. logf "sigs.k8s.io/controller-runtime/pkg/log"
  33. "sigs.k8s.io/controller-runtime/pkg/reconcile"
  34. "github.com/LocoStack/loco-operator/api/v1alpha1"
  35. "github.com/LocoStack/loco-operator/internal/reconciler"
  36. "github.com/LocoStack/loco-operator/pkg/templates"
  37. )
  38. // StackReconciler reconciles a Stack object
  39. type StackReconciler struct {
  40. client.Client
  41. Scheme *runtime.Scheme
  42. }
  43. // +kubebuilder:rbac:groups=locostack.com,resources=stacks,verbs=get;list;watch;create;update;patch;delete
  44. // +kubebuilder:rbac:groups=locostack.com,resources=stacks/status,verbs=get;update;patch
  45. // +kubebuilder:rbac:groups=locostack.com,resources=stacks/finalizers,verbs=update
  46. // +kubebuilder:rbac:groups=locostack.com,resources=components,verbs=get;list;watch;create;update;patch;delete
  47. // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get;list;watch;create;update;patch
  48. // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch
  49. func (r *StackReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
  50. log := logf.FromContext(ctx)
  51. s := &v1alpha1.Stack{}
  52. if err := r.Get(ctx, req.NamespacedName, s); err != nil {
  53. if apierrors.IsNotFound(err) {
  54. return ctrl.Result{}, nil
  55. }
  56. return ctrl.Result{}, err
  57. }
  58. patch := client.MergeFrom(s.DeepCopy())
  59. if err := r.reconcileSharedVolumes(ctx, s); err != nil {
  60. return ctrl.Result{}, err
  61. }
  62. if err := r.reconcileComponents(ctx, s); err != nil {
  63. return ctrl.Result{}, err
  64. }
  65. if err := r.reconcileGateways(ctx, s); err != nil {
  66. return ctrl.Result{}, err
  67. }
  68. s.Status.ObservedGeneration = s.Generation
  69. if err := r.Status().Patch(ctx, s, patch); err != nil {
  70. return ctrl.Result{}, client.IgnoreNotFound(err)
  71. }
  72. log.Info("Reconciled Stack", "namespace", s.Namespace, "name", s.Name)
  73. return ctrl.Result{}, nil
  74. }
  75. // reconcileSharedVolumes ensures all PVC-backed shared volumes declared in the Stack spec exist.
  76. func (r *StackReconciler) reconcileSharedVolumes(ctx context.Context, s *v1alpha1.Stack) error {
  77. failedVolumes := make([]string, 0)
  78. for _, volume := range s.Spec.SharedVolumes {
  79. claimName := fmt.Sprintf("%s-%s", s.Name, volume)
  80. pvc := &corev1.PersistentVolumeClaim{
  81. ObjectMeta: metav1.ObjectMeta{
  82. Name: claimName,
  83. Namespace: s.Namespace,
  84. },
  85. }
  86. pv := &corev1.PersistentVolume{}
  87. if err := r.Get(ctx, client.ObjectKey{Name: volume}, pv); err != nil {
  88. if apierrors.IsNotFound(err) {
  89. apimeta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{
  90. Type: "SharedVolumesResolved",
  91. Status: metav1.ConditionFalse,
  92. Reason: "ReconcileFailed",
  93. Message: fmt.Sprintf("Could not find shared volume %s: %v", volume, err),
  94. ObservedGeneration: s.Generation,
  95. })
  96. return nil
  97. }
  98. return err
  99. }
  100. _, err := controllerutil.CreateOrUpdate(ctx, r.Client, pvc, func() error {
  101. if pvc.Labels == nil {
  102. pvc.Labels = map[string]string{}
  103. }
  104. maps.Copy(pvc.Labels, reconciler.ResourceLabels(s.Name, "SharedVolume", volume, ""))
  105. pvc.Spec.VolumeName = volume
  106. pvc.Spec.StorageClassName = ptr.To("")
  107. pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}
  108. if pvc.Spec.Resources.Requests == nil {
  109. pvc.Spec.Resources.Requests = corev1.ResourceList{}
  110. }
  111. if _, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; !ok {
  112. pvc.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("1Gi")
  113. }
  114. return controllerutil.SetControllerReference(s, pvc, r.Scheme)
  115. })
  116. if err != nil {
  117. failedVolumes = append(failedVolumes, volume)
  118. }
  119. }
  120. if len(failedVolumes) > 0 {
  121. apimeta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{
  122. Type: "SharedVolumesResolved",
  123. Status: metav1.ConditionFalse,
  124. Reason: "ReconcileFailed",
  125. Message: fmt.Sprintf("Could not reconcile shared volume(s): %v", failedVolumes),
  126. ObservedGeneration: s.Generation,
  127. })
  128. } else {
  129. apimeta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{
  130. Type: "SharedVolumesResolved",
  131. Status: metav1.ConditionTrue,
  132. Reason: "ReconcileSucceeded",
  133. Message: "All shared volumes reconciled successfully",
  134. ObservedGeneration: s.Generation,
  135. })
  136. }
  137. return nil
  138. }
  139. func (r *StackReconciler) reconcileComponents(ctx context.Context, s *v1alpha1.Stack) error {
  140. components := getComponents(s)
  141. failedComponents := make([]string, 0)
  142. errList := make([]error, 0)
  143. for _, comp := range components {
  144. resolvedTemplateName, reconciled, err := r.reconcileComponent(ctx, s, comp)
  145. setResolvedTemplateName(&s.Status, comp.name, resolvedTemplateName)
  146. if !reconciled {
  147. failedComponents = append(failedComponents, comp.name)
  148. }
  149. if err != nil {
  150. errList = append(errList, err)
  151. }
  152. }
  153. if len(failedComponents) > 0 {
  154. apimeta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{
  155. Type: "ComponentsReconciled",
  156. Status: metav1.ConditionFalse,
  157. Reason: "ReconcileFailed",
  158. Message: fmt.Sprintf("%d component(s) failed to reconcile: %v", len(failedComponents), failedComponents),
  159. ObservedGeneration: s.Generation,
  160. })
  161. } else {
  162. apimeta.SetStatusCondition(&s.Status.Conditions, metav1.Condition{
  163. Type: "ComponentsReconciled",
  164. Status: metav1.ConditionTrue,
  165. Reason: "ReconcileSucceeded",
  166. Message: "All enabled components were reconciled successfully",
  167. ObservedGeneration: s.Generation,
  168. })
  169. }
  170. if len(errList) > 0 {
  171. return fmt.Errorf("failed with errors: %v", errList)
  172. }
  173. return nil
  174. }
  175. type componentConfig struct {
  176. name string
  177. defaultTmpl string
  178. template *v1alpha1.Template
  179. enabled bool
  180. }
  181. func newComponentConfig(name, defaultTmpl string, enabled bool, optional *v1alpha1.OptionalComponent) componentConfig {
  182. var tmpl *v1alpha1.Template
  183. if optional != nil {
  184. enabled = optional.Enabled
  185. tmpl = optional.Template
  186. }
  187. return componentConfig{
  188. name: name,
  189. defaultTmpl: defaultTmpl,
  190. template: tmpl,
  191. enabled: enabled,
  192. }
  193. }
  194. func getComponents(s *v1alpha1.Stack) []componentConfig {
  195. return []componentConfig{
  196. newComponentConfig("Gateway", "litellm", true, s.Spec.Gateway),
  197. newComponentConfig("VectorStore", "qdrant", true, s.Spec.VectorStore),
  198. newComponentConfig("GraphStore", "neo4j", false, s.Spec.GraphStore),
  199. newComponentConfig("Database", "postgresql", true, s.Spec.Database),
  200. newComponentConfig("Observability", "phoenix", false, s.Spec.Observability),
  201. }
  202. }
  203. func setResolvedTemplateName(status *v1alpha1.StackStatus, name string, templateName string) {
  204. switch name {
  205. case "Gateway":
  206. status.ComponentStatus.Gateway = templateName
  207. case "VectorStore":
  208. status.ComponentStatus.VectorStore = templateName
  209. case "GraphStore":
  210. status.ComponentStatus.GraphStore = templateName
  211. case "Database":
  212. status.ComponentStatus.Database = templateName
  213. case "Observability":
  214. status.ComponentStatus.Observability = templateName
  215. }
  216. }
  217. func (r *StackReconciler) reconcileComponent(ctx context.Context, s *v1alpha1.Stack, compCfg componentConfig) (string, bool, error) {
  218. compName := fmt.Sprintf("%s-%s", strings.ToLower(compCfg.name), s.Name)
  219. comp := &v1alpha1.Component{
  220. ObjectMeta: metav1.ObjectMeta{
  221. Name: compName,
  222. Namespace: s.Namespace,
  223. },
  224. }
  225. err := r.Get(ctx, client.ObjectKey{Namespace: s.Namespace, Name: compName}, comp)
  226. if !compCfg.enabled {
  227. tmpl := "disabled"
  228. if err == nil {
  229. if err := r.Delete(ctx, comp); err != nil {
  230. return tmpl, false, err
  231. }
  232. return tmpl, true, nil
  233. }
  234. if apierrors.IsNotFound(err) {
  235. return tmpl, true, nil
  236. }
  237. return tmpl, false, err
  238. }
  239. tmpl, err := templates.Manager.ResolveTemplate(compCfg.template, compCfg.defaultTmpl)
  240. if err != nil {
  241. return "unknown", false, err
  242. }
  243. _, err = controllerutil.CreateOrUpdate(ctx, r.Client, comp, func() error {
  244. if comp.Labels == nil {
  245. comp.Labels = make(map[string]string)
  246. }
  247. maps.Copy(comp.Labels, reconciler.ResourceLabels(s.Name, compCfg.name, s.Name, ""))
  248. comp.Labels["stack.locostack.com/default"] = "true"
  249. comp.Spec.Category = compCfg.name
  250. comp.Spec.Template = &tmpl
  251. comp.Spec.StackRef = &corev1.LocalObjectReference{Name: s.Name}
  252. return controllerutil.SetControllerReference(s, comp, r.Scheme)
  253. })
  254. available := apimeta.IsStatusConditionTrue(comp.Status.Conditions, "Available")
  255. return tmpl.Name, available, err
  256. }
  257. func (r *StackReconciler) reconcileGateways(ctx context.Context, s *v1alpha1.Stack) error {
  258. log := logf.FromContext(ctx)
  259. gwList := &v1alpha1.ComponentList{}
  260. err := r.List(ctx, gwList, client.InNamespace(s.Namespace), client.MatchingLabels{"locostack.com/stack": s.Name, "locostack.com/component": "Gateway"})
  261. if err != nil {
  262. return fmt.Errorf("failed to list gateway components: %v", err)
  263. }
  264. if len(gwList.Items) == 0 {
  265. log.Info("No gateways found for stack", "namespace", s.Namespace, "stack", s.Name)
  266. return nil
  267. }
  268. dependencies := make([]corev1.ObjectReference, 0)
  269. emList := &v1alpha1.ExternalModelList{}
  270. if err := r.List(ctx, emList, client.InNamespace(s.Namespace)); err != nil {
  271. return fmt.Errorf("Failed to list external models: %v", err)
  272. }
  273. for _, em := range emList.Items {
  274. if em.Spec.StackRef.Name == s.Name {
  275. dependencies = append(dependencies, corev1.ObjectReference{
  276. Kind: "ExternalModel",
  277. Name: em.Name,
  278. Namespace: em.Namespace,
  279. })
  280. }
  281. }
  282. mmList := &v1alpha1.ManagedModelList{}
  283. if err := r.List(ctx, mmList, client.InNamespace(s.Namespace)); err != nil {
  284. return fmt.Errorf("Failed to list managed models: %v", err)
  285. }
  286. for _, em := range mmList.Items {
  287. if em.Spec.StackRef.Name == s.Name {
  288. dependencies = append(dependencies, corev1.ObjectReference{
  289. Kind: "ManagedModel",
  290. Name: em.Name,
  291. Namespace: em.Namespace,
  292. })
  293. }
  294. }
  295. etList := &v1alpha1.ExternalToolList{}
  296. if err := r.List(ctx, etList, client.InNamespace(s.Namespace)); err != nil {
  297. return fmt.Errorf("Failed to list external tools: %v", err)
  298. }
  299. for _, em := range etList.Items {
  300. if em.Spec.StackRef.Name == s.Name {
  301. dependencies = append(dependencies, corev1.ObjectReference{
  302. Kind: "ExternalTool",
  303. Name: em.Name,
  304. Namespace: em.Namespace,
  305. })
  306. }
  307. }
  308. mtList := &v1alpha1.ManagedToolList{}
  309. if err := r.List(ctx, mtList, client.InNamespace(s.Namespace)); err != nil {
  310. return fmt.Errorf("Failed to list managed tools: %v", err)
  311. }
  312. for _, em := range mtList.Items {
  313. if em.Spec.StackRef.Name == s.Name {
  314. dependencies = append(dependencies, corev1.ObjectReference{
  315. Kind: "ManagedTool",
  316. Name: em.Name,
  317. Namespace: em.Namespace,
  318. })
  319. }
  320. }
  321. for i := range gwList.Items {
  322. gw := &gwList.Items[i]
  323. // written even when the Stack has no models or tools left: the gateway would
  324. // otherwise keep serving - and keep failing to resolve - deleted dependencies
  325. if equality.Semantic.DeepEqual(gw.Spec.Dependencies, dependencies) {
  326. continue
  327. }
  328. gw.Spec.Dependencies = dependencies
  329. if err := r.Update(ctx, gw); err != nil {
  330. return fmt.Errorf("Failed to update gateway %s with dependencies: %v", gw.Name, err)
  331. }
  332. }
  333. return nil
  334. }
  335. // SetupWithManager sets up the controller with the Manager.
  336. func (r *StackReconciler) SetupWithManager(mgr ctrl.Manager) error {
  337. mapToStack := func(ctx context.Context, obj client.Object) []reconcile.Request {
  338. var stackName string
  339. switch typed := obj.(type) {
  340. case *v1alpha1.ExternalModel:
  341. stackName = typed.Spec.StackRef.Name
  342. case *v1alpha1.ManagedModel:
  343. stackName = typed.Spec.StackRef.Name
  344. case *v1alpha1.ExternalTool:
  345. stackName = typed.Spec.StackRef.Name
  346. case *v1alpha1.ManagedTool:
  347. stackName = typed.Spec.StackRef.Name
  348. default:
  349. return nil
  350. }
  351. stackList := &v1alpha1.StackList{}
  352. if err := mgr.GetClient().List(ctx, stackList, client.InNamespace(obj.GetNamespace())); err != nil {
  353. return nil
  354. }
  355. var reqs []reconcile.Request
  356. for _, stack := range stackList.Items {
  357. if stackName == stack.Name {
  358. reqs = append(reqs, reconcile.Request{
  359. NamespacedName: types.NamespacedName{Name: stack.Name, Namespace: stack.Namespace},
  360. })
  361. }
  362. }
  363. return reqs
  364. }
  365. return ctrl.NewControllerManagedBy(mgr).
  366. For(&v1alpha1.Stack{}).
  367. Owns(&v1alpha1.Component{}).
  368. Owns(&corev1.PersistentVolumeClaim{}).
  369. Watches(&v1alpha1.ExternalModel{}, handler.EnqueueRequestsFromMapFunc(mapToStack)).
  370. Watches(&v1alpha1.ManagedModel{}, handler.EnqueueRequestsFromMapFunc(mapToStack)).
  371. Watches(&v1alpha1.ExternalTool{}, handler.EnqueueRequestsFromMapFunc(mapToStack)).
  372. Watches(&v1alpha1.ManagedTool{}, handler.EnqueueRequestsFromMapFunc(mapToStack)).
  373. Named("stack").
  374. Complete(r)
  375. }