loco-operator/pkg/templates/litellm/config.go

/*
Copyright 2026 LocoStack.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package litellm

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/LocoStack/loco-operator/api/v1alpha1"
)

type LiteLLMConfig struct {
	ModelList       []modelEntry              `yaml:"model_list"`
	MCPServers      map[string]mcpServerEntry `yaml:"mcp_servers,omitempty"`
	LiteLLMSettings *liteLLMSettings          `yaml:"litellm_settings,omitempty"`
	GeneralSettings *liteLLMGeneralSettings   `yaml:"general_settings,omitempty"`
	EnvVars         map[string]string         `yaml:"environment_variables,omitempty"`
}

type modelEntry struct {
	ModelName     string         `yaml:"model_name"`
	LiteLLMParams map[string]any `yaml:"litellm_params"`
}

type mcpServerEntry struct {
	// url is the HTTP/SSE endpoint for sse and http transports.
	URL string `yaml:"url,omitempty"`
	// transport selects the MCP transport: sse (default), http, or stdio.
	Transport string `yaml:"transport,omitempty"`

	// --- authentication ---

	// authType selects the auth method: api_key, bearer_token, basic,
	// authorization, oauth2, or aws_sigv4.
	AuthType string `yaml:"auth_type,omitempty"`
	// authValue is the credential for api_key, bearer_token, basic, and
	// authorization auth types. May reference an env var via os.environ/VAR.
	AuthValue string `yaml:"auth_value,omitempty"`

	// --- headers ---

	// staticHeaders are key-value pairs sent with every request to this server.
	StaticHeaders map[string]string `yaml:"static_headers,omitempty"`
	// extraHeaders lists client request header names that LiteLLM should
	// forward to this MCP server.
	ExtraHeaders []string `yaml:"extra_headers,omitempty"`

	// allowAllKeys grants every LiteLLM API key access to this server when true.
	AllowAllKeys bool `yaml:"allow_all_keys,omitempty"`
}

type liteLLMSettings struct {
	SuccessCallback []string `yaml:"success_callback,omitempty"`
	FailureCallback []string `yaml:"failure_callback,omitempty"`
	JSONLogs        bool     `yaml:"json_logs,omitempty"`
}

type liteLLMGeneralSettings struct {
	MasterKey string `yaml:"master_key,omitempty"`
}

type LiteLLMConfigBuilder struct {
	MasterKeyEnvName       string
	Stack                  *v1alpha1.Stack
	ObservabilityComponent *v1alpha1.Component
	ExternalModels         []*v1alpha1.ExternalModel
	ManagedModels          []*v1alpha1.ManagedModel
}

func (l *LiteLLMConfigBuilder) BuildLiteLLMConfig() (LiteLLMConfig, error) {
	modelList := []modelEntry{}
	for _, em := range l.ExternalModels {
		entry := buildExternalModelEntry(em)
		modelList = append(modelList, entry)
	}
	for _, mm := range l.ManagedModels {
		if mm.Status.Endpoint == "" {
			continue
		}
		entry := buildManagedModelEntry(mm)
		modelList = append(modelList, entry)
	}
	phoenixEnabled := phoenixEnabled(l.Stack)
	successCB := []string{}
	failureCB := []string{}
	if phoenixEnabled {
		successCB = append(successCB, "arize_phoenix")
		failureCB = append(failureCB, "arize_phoenix")
	}
	cfg := LiteLLMConfig{
		ModelList:  modelList,
		MCPServers: map[string]mcpServerEntry{},
		LiteLLMSettings: &liteLLMSettings{
			SuccessCallback: successCB,
			FailureCallback: failureCB,
			JSONLogs:        true,
		},
		GeneralSettings: &liteLLMGeneralSettings{
			MasterKey: "os.environ/" + l.MasterKeyEnvName,
		},
	}
	if phoenixEnabled {
		cfg.EnvVars = map[string]string{
			"PHOENIX_PROJECT_NAME":            "locostack",
			"PHOENIX_COLLECTOR_HTTP_ENDPOINT": l.ObservabilityComponent.Status.Endpoint + "/v1/traces",
		}
	}
	return cfg, nil
}

func phoenixEnabled(stack *v1alpha1.Stack) bool {
	if stack == nil {
		return false
	}
	if stack.Spec.Observability == nil {
		return false
	}
	if !stack.Spec.Observability.Enabled {
		return false
	}
	if stack.Spec.Observability.Template != nil && stack.Spec.Observability.Template.Name != "phoenix" {
		return false
	}
	return true
}

func AuthEnvVarName(kind string, name string) string {
	return fmt.Sprintf("%s_%s_AUTH", strings.ToUpper(kind), strings.ToUpper(strings.ReplaceAll(name, "-", "_")))
}

// buildExternalModelEntry converts an ExternalModel into a LiteLLM model_list entry.
func buildExternalModelEntry(em *v1alpha1.ExternalModel) modelEntry {
	params := map[string]any{
		"model": fmt.Sprintf("%s/%s", em.Spec.Provider, em.Spec.ProviderModel),
	}
	if em.Spec.Auth != nil && (em.Spec.Auth.APIKey != nil || em.Spec.Auth.BearerToken != nil) {
		params["api_key"] = "os.environ/" + AuthEnvVarName("ExternalModel", em.Name)
	}
	if em.Spec.APIBase != "" {
		params["api_base"] = em.Spec.APIBase
	}
	if em.Spec.APIVersion != "" {
		params["api_version"] = em.Spec.APIVersion
	}
	switch em.Spec.Category {
	case "embedding":
		params["mode"] = "embedding"
	case "reranker":
		params["mode"] = "rerank"
	}
	for k, v := range em.Spec.ExtraParams {
		params[k] = v
	}
	if em.Spec.DefaultInferenceParams != nil {
		applyInferenceParams(params, em.Spec.DefaultInferenceParams)
	}
	return modelEntry{
		ModelName:     em.Spec.ModelName,
		LiteLLMParams: params,
	}
}

// buildManagedModelEntry converts a ManagedModel into a LiteLLM model_list entry.
func buildManagedModelEntry(mm *v1alpha1.ManagedModel) modelEntry {
	// litellm's rerank dispatch does not recognize "openai" as a provider;
	// "hosted_vllm" is the generic self-hosted provider for the /rerank call type.
	provider := "openai"
	if mm.Spec.Category == "reranker" {
		provider = "hosted_vllm"
	}
	params := map[string]any{
		"model":    fmt.Sprintf("%s/%s", provider, mm.Spec.ModelName),
		"api_base": mm.Status.Endpoint + "/v1",
		"api_key":  "not-set",
	}
	switch mm.Spec.Category {
	case "embedding":
		params["mode"] = "embedding"
	case "reranker":
		params["mode"] = "rerank"
	}
	return modelEntry{
		ModelName:     mm.Spec.ModelName,
		LiteLLMParams: params,
	}
}

func applyInferenceParams(params map[string]any, inf *v1alpha1.InferenceParameters) {
	if inf.Temperature != "" {
		if v, err := strconv.ParseFloat(inf.Temperature, 64); err == nil {
			params["temperature"] = v
		}
	}
	if inf.TopP != "" {
		if v, err := strconv.ParseFloat(inf.TopP, 64); err == nil {
			params["top_p"] = v
		}
	}
	if inf.MaxTokens != nil {
		params["max_tokens"] = *inf.MaxTokens
	}
	for k, v := range inf.Extra {
		params[k] = v
	}
}