loco-operator/internal/reconciler/litellm.go

/*
Copyright 2026 LocoStack.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reconciler

import (
	"context"
	"crypto/rand"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"maps"

	"github.com/LocoStack/loco-operator/api/v1alpha1"
	"github.com/LocoStack/loco-operator/pkg/templates/litellm"
	"go.yaml.in/yaml/v2"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

type LiteLLMReconciler struct {
	*DefaultComponentReconciler
	client    client.Client
	scheme    *runtime.Scheme
	stack     *v1alpha1.Stack
	component *v1alpha1.Component
}

func NewLiteLLMReconciler(client client.Client, scheme *runtime.Scheme, stack *v1alpha1.Stack, component *v1alpha1.Component) *LiteLLMReconciler {
	return &LiteLLMReconciler{
		DefaultComponentReconciler: NewDefaultComponentReconciler(client, scheme, stack, component),
		client:                     client,
		scheme:                     scheme,
		stack:                      stack,
		component:                  component,
	}
}

func (r *LiteLLMReconciler) ReconcileComponent(ctx context.Context, tmpl *v1alpha1.Template, variables map[string]string) ([]client.Object, error) {
	if err := r.ReconcileKey(ctx, "litellm", litellm.LITELLM_AUTH_SECRET_KEY, generateKey); err != nil {
		return nil, fmt.Errorf("Failed to reconcile LiteLLM master key: %w", err)
	}
	deps := make([]client.Object, 0)
	emList := make([]*v1alpha1.ExternalModel, 0)
	mmList := make([]*v1alpha1.ManagedModel, 0)
	etList := make([]*v1alpha1.ExternalTool, 0)
	mtList := make([]*v1alpha1.ManagedTool, 0)
	for _, dep := range r.component.Spec.Dependencies {
		switch dep.Kind {
		case "ExternalModel":
			em := &v1alpha1.ExternalModel{}
			if err := r.client.Get(ctx, client.ObjectKey{Name: dep.Name, Namespace: dep.Namespace}, em); err != nil {
				return nil, fmt.Errorf("Failed to get ExternalModel %s/%s: %w", dep.Namespace, dep.Name, err)
			}
			emList = append(emList, em)
			deps = append(deps, em)
		case "ManagedModel":
			mm := &v1alpha1.ManagedModel{}
			if err := r.client.Get(ctx, client.ObjectKey{Name: dep.Name, Namespace: dep.Namespace}, mm); err != nil {
				return nil, fmt.Errorf("Failed to get ManagedModel %s/%s: %w", dep.Namespace, dep.Name, err)
			}
			mmList = append(mmList, mm)
			deps = append(deps, mm)
		case "ExternalTool":
			et := &v1alpha1.ExternalTool{}
			if err := r.client.Get(ctx, client.ObjectKey{Name: dep.Name, Namespace: dep.Namespace}, et); err != nil {
				return nil, fmt.Errorf("Failed to get ExternalTool %s/%s: %w", dep.Namespace, dep.Name, err)
			}
			etList = append(etList, et)
			deps = append(deps, et)
		case "ManagedTool":
			mt := &v1alpha1.ManagedTool{}
			if err := r.client.Get(ctx, client.ObjectKey{Name: dep.Name, Namespace: dep.Namespace}, mt); err != nil {
				return nil, fmt.Errorf("Failed to get ManagedTool %s/%s: %w", dep.Namespace, dep.Name, err)
			}
			mtList = append(mtList, mt)
			deps = append(deps, mt)
		}
	}
	configHash, err := r.reconcileConfigMap(ctx, emList, mmList, etList, mtList)
	if err != nil {
		return nil, fmt.Errorf("Failed to reconcile LiteLLM config: %w", err)
	}
	authSpecs := make(map[string]map[string]*v1alpha1.AuthSpec)
	for _, em := range emList {
		if em.Spec.Auth != nil {
			if _, ok := authSpecs["ExternalModel"]; !ok {
				authSpecs["ExternalModel"] = make(map[string]*v1alpha1.AuthSpec)
			}
			authSpecs["ExternalModel"][em.Name] = em.Spec.Auth
		}
	}
	for _, et := range etList {
		if et.Spec.Auth != nil {
			if _, ok := authSpecs["ExternalTool"]; !ok {
				authSpecs["ExternalTool"] = make(map[string]*v1alpha1.AuthSpec)
			}
			authSpecs["ExternalTool"][et.Name] = et.Spec.Auth
		}
	}
	for _, mt := range mtList {
		if mt.Spec.Auth != nil {
			if _, ok := authSpecs["ManagedTool"]; !ok {
				authSpecs["ManagedTool"] = make(map[string]*v1alpha1.AuthSpec)
			}
			authSpecs["ManagedTool"][mt.Name] = mt.Spec.Auth
		}
	}
	credentialsHash, err := r.credentialsHash(ctx, authSpecs)
	if err != nil {
		return nil, fmt.Errorf("Failed to hash LiteLLM credentials: %w", err)
	}
	injectSecrets(tmpl, authSpecs)
	if tmpl.Metadata.Annotations == nil {
		tmpl.Metadata.Annotations = make(map[string]string)
	}
	// the credential values reach the Pod as env vars, which are snapshotted when it
	// starts, so the rendered config alone does not decide whether the gateway is current:
	// a key rotated in place has to roll it out as well
	h := sha256.Sum256([]byte(configHash + credentialsHash))
	maps.Copy(tmpl.Metadata.Annotations, map[string]string{
		"locostack.com/configHash": hex.EncodeToString(h[:]),
	})
	if _, err := r.DefaultComponentReconciler.ReconcileComponent(ctx, tmpl, variables); err != nil {
		return nil, err
	}
	return deps, nil
}

func (r *LiteLLMReconciler) reconcileConfigMap(ctx context.Context, emList []*v1alpha1.ExternalModel, mmList []*v1alpha1.ManagedModel, etList []*v1alpha1.ExternalTool, mtList []*v1alpha1.ManagedTool) (string, error) {
	var o11yComp *v1alpha1.Component
	if r.stack.Spec.Observability != nil && r.stack.Spec.Observability.Enabled {
		o11yComp = &v1alpha1.Component{}
		if err := r.client.Get(ctx, client.ObjectKey{Name: fmt.Sprintf("observability-%s", r.stack.Name), Namespace: r.stack.Namespace}, o11yComp); err != nil {
			return "", fmt.Errorf("Failed to get Observability component: %w", err)
		}
	}
	configBuilder := litellm.LiteLLMConfigBuilder{
		MasterKeyEnvName:       litellm.LITELLM_MASTER_KEY_ENV_NAME,
		Stack:                  r.stack,
		ObservabilityComponent: o11yComp,
		ExternalModels:         emList,
		ManagedModels:          mmList,
		ExternalTools:          etList,
		ManagedTools:           mtList,
	}
	config, err := configBuilder.BuildLiteLLMConfig()
	if err != nil {
		return "", fmt.Errorf("Failed to build LiteLLM config: %w", err)
	}
	configData, err := yaml.Marshal(config)
	if err != nil {
		return "", fmt.Errorf("Failed to marshal LiteLLM config: %w", err)
	}
	configDataStr := string(configData)
	h := sha256.Sum256([]byte(configDataStr))
	configHash := hex.EncodeToString(h[:])
	cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: r.ResourceName("config"), Namespace: r.component.GetNamespace()}}
	if _, err := controllerutil.CreateOrUpdate(ctx, r.client, cm, func() error {
		if cm.Labels == nil {
			cm.Labels = map[string]string{}
		}
		maps.Copy(cm.Labels, r.ResourceLabels())
		if cm.Annotations == nil {
			cm.Annotations = make(map[string]string)
		}
		cm.Annotations["hash"] = configHash
		cm.Data = map[string]string{litellm.LITELLM_CONFIG_KEY: configDataStr}
		return controllerutil.SetControllerReference(r.component, cm, r.scheme)
	}); err != nil {
		return "", err
	}
	return configHash, nil
}

// credentialsHash digests the credential values that the gateway Pod reads from Secrets,
// keyed by the dependency and Secret key they belong to. A Secret or key that does not
// resolve hashes as empty rather than failing the reconcile: the rest of the config still
// has to be written, and it is the dependency's own CredentialsResolved condition that
// reports the dangling reference.
func (r *LiteLLMReconciler) credentialsHash(ctx context.Context, authSpecs map[string]map[string]*v1alpha1.AuthSpec) (string, error) {
	h := sha256.New()
	for _, kind := range sortedKeys(authSpecs) {
		for _, name := range sortedKeys(authSpecs[kind]) {
			for _, ref := range AuthSecretRefs(authSpecs[kind][name]) {
				secret := &corev1.Secret{}
				if err := r.client.Get(ctx, client.ObjectKey{Name: ref.Name, Namespace: r.component.GetNamespace()}, secret); err != nil && !apierrors.IsNotFound(err) {
					return "", fmt.Errorf("Failed to get Secret %s: %w", ref.Name, err)
				}
				_, _ = fmt.Fprintf(h, "%s/%s/%s/%s=%s\n", kind, name, ref.Name, ref.Key, secret.Data[ref.Key])
			}
		}
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

func generateKey() (string, error) {
	b := make([]byte, 24)
	if _, err := rand.Read(b); err != nil {
		return "", err
	}
	return "sk-" + hex.EncodeToString(b), nil
}

func injectSecrets(tmpl *v1alpha1.Template, authSpecs map[string]map[string]*v1alpha1.AuthSpec) {
	envVars := []corev1.EnvVar{}
	for _, kind := range sortedKeys(authSpecs) {
		kindAuthSpecs := authSpecs[kind]
		for _, name := range sortedKeys(kindAuthSpecs) {
			auth := kindAuthSpecs[name]
			var refName string
			var refKey string
			if auth.APIKey != nil {
				refName = auth.APIKey.SecretRef.Name
				refKey = auth.APIKey.SecretRef.Key
			} else if auth.BearerToken != nil {
				refName = auth.BearerToken.Name
				refKey = auth.BearerToken.Key
			}
			if refName != "" && refKey != "" {
				envVars = append(envVars, corev1.EnvVar{
					Name: litellm.AuthEnvVarName(kind, name),
					ValueFrom: &corev1.EnvVarSource{
						SecretKeyRef: &corev1.SecretKeySelector{
							LocalObjectReference: corev1.LocalObjectReference{Name: refName},
							Key:                  refKey,
						},
					},
				})
			}
			if auth.Headers != nil {
				for _, header := range auth.Headers {
					if header.ValueFrom.SecretKeyRef != nil {
						envVars = append(envVars, corev1.EnvVar{
							Name: litellm.AuthEnvVarName(kind, name+"_"+header.Name),
							ValueFrom: &corev1.EnvVarSource{
								SecretKeyRef: &corev1.SecretKeySelector{
									LocalObjectReference: header.ValueFrom.SecretKeyRef.LocalObjectReference,
									Key:                  header.ValueFrom.SecretKeyRef.Key,
								},
							},
						})
					}
				}
			}
		}
	}
	tmpl.Spec.Runtime.Env = append(tmpl.Spec.Runtime.Env, envVars...)
}