| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436 |
- import { Cause, Context, Effect, Layer, Schema, Stream } from "effect"
- import * as Option from "effect/Option"
- import { Auth, type Auth as AuthDef } from "./auth"
- import { Endpoint, type EndpointPatch } from "./endpoint"
- import { RequestExecutor } from "./executor"
- import type { Framing } from "./framing"
- import { HttpTransport } from "./transport"
- import type { Transport, TransportRuntime } from "./transport"
- import { WebSocketExecutor } from "./transport"
- import type { Protocol } from "./protocol"
- import { applyCachePolicy } from "../cache-policy"
- import * as ProviderShared from "../protocols/shared"
- import type { LLMError, LLMEvent, PreparedRequestOf, ProtocolID, ProviderOptions } from "../schema"
- import {
- GenerationOptions,
- HttpOptions,
- LLMRequest,
- LLMResponse,
- Model,
- ModelLimits,
- LLMError as LLMErrorClass,
- PreparedRequest,
- ProviderID,
- mergeGenerationOptions,
- mergeHttpOptions,
- mergeProviderOptions,
- } from "../schema"
- export interface RouteBody<Body> {
- /** Schema for the validated provider-native body sent as the JSON request. */
- readonly schema: Schema.Codec<Body, unknown>
- /** Build the provider-native body from a common `LLMRequest`. */
- readonly from: (request: LLMRequest) => Effect.Effect<Body, LLMError>
- }
- export interface Route<Body, Prepared = unknown> {
- readonly id: string
- readonly provider?: ProviderID
- readonly protocol: ProtocolID
- readonly endpoint: Endpoint<Body>
- readonly auth: AuthDef
- readonly transport: Transport<Body, Prepared, unknown>
- readonly defaults: RouteDefaults
- readonly body: RouteBody<Body>
- readonly with: (patch: RoutePatch<Body, Prepared>) => Route<Body, Prepared>
- readonly model: (input: RouteMappedModelInput) => Model
- readonly prepareTransport: (body: Body, request: LLMRequest) => Effect.Effect<Prepared, LLMError>
- readonly streamPrepared: (
- prepared: Prepared,
- request: LLMRequest,
- runtime: TransportRuntime,
- ) => Stream.Stream<LLMEvent, LLMError>
- }
- // Route registries intentionally erase body generics after construction.
- // Normal call sites use `OpenAIChat.route`; callers only need body types
- // when preparing a request with a protocol-specific type assertion.
- // oxlint-disable-next-line typescript-eslint/no-explicit-any
- export type AnyRoute = Route<any, any>
- export type HttpOptionsInput = HttpOptions.Input
- export type RouteModelInput = Omit<Model.Input, "provider" | "route">
- export type RouteRoutedModelInput = Omit<Model.Input, "route">
- export interface RouteDefaults {
- readonly headers?: Record<string, string>
- readonly limits?: ModelLimits
- readonly generation?: GenerationOptions
- readonly providerOptions?: ProviderOptions
- readonly http?: HttpOptions
- }
- export interface RouteDefaultsInput {
- readonly headers?: Record<string, string>
- readonly limits?: ModelLimits.Input
- readonly generation?: GenerationOptions.Input
- readonly providerOptions?: ProviderOptions
- readonly http?: HttpOptions.Input
- }
- export interface RoutePatch<Body, Prepared> extends RouteDefaultsInput {
- readonly id?: string
- readonly provider?: string | ProviderID
- readonly auth?: AuthDef
- readonly transport?: Transport<Body, Prepared, unknown>
- readonly endpoint?: EndpointPatch<Body>
- }
- type RouteMappedModelInput = RouteModelInput | RouteRoutedModelInput
- const makeRouteModel = (route: AnyRoute, mapped: RouteMappedModelInput) => {
- const provider = route.provider ?? ("provider" in mapped ? mapped.provider : undefined)
- if (!provider) throw new Error(`Route.model(${route.id}) requires a provider`)
- if (!endpointBaseURL(route.endpoint))
- throw new Error(`Route.model(${route.id}) requires an endpoint baseURL — configure it on the route first`)
- return Model.make({
- ...mapped,
- provider,
- route,
- })
- }
- const mergeRouteDefaults = (base: RouteDefaults | undefined, patch: RouteDefaultsInput): RouteDefaults => {
- const headers = mergeHeaders(base?.headers, patch.headers)
- return {
- ...base,
- ...patch,
- headers,
- limits: patch.limits === undefined ? base?.limits : ModelLimits.make(patch.limits),
- generation: mergeGenerationOptions(generationOptions(base?.generation), generationOptions(patch.generation)),
- providerOptions: mergeProviderOptions(base?.providerOptions, patch.providerOptions),
- http: mergeHttpOptions(
- base?.http,
- httpOptions(patch.http),
- headers === undefined ? undefined : new HttpOptions({ headers }),
- ),
- }
- }
- const endpointBaseURL = <Body>(endpoint: Endpoint<Body>) =>
- typeof endpoint.baseURL === "string" ? endpoint.baseURL : undefined
- const mergeHeaders = (...items: ReadonlyArray<Record<string, string> | undefined>) => {
- const entries = items.flatMap((item) =>
- item === undefined ? [] : Object.entries(item).filter((entry): entry is [string, string] => entry[1] !== undefined),
- )
- if (entries.length === 0) return undefined
- return Object.fromEntries(entries)
- }
- export const generationOptions = (input: GenerationOptions.Input | undefined) =>
- input === undefined ? undefined : GenerationOptions.make(input)
- export const httpOptions = (input: HttpOptionsInput | undefined) => {
- if (input === undefined) return input
- return HttpOptions.make(input)
- }
- export interface Interface {
- /**
- * Compile a request through protocol body construction, validation, and HTTP
- * preparation without sending it. Returns the prepared request including the
- * provider-native body.
- *
- * Pass a `Body` type argument to statically expose the route's body
- * shape (e.g. `prepare<OpenAIChatBody>(...)`) — the runtime body is
- * identical, so this is a type-level assertion the caller makes about which
- * route the request will resolve to.
- */
- readonly prepare: <Body = unknown>(request: LLMRequest) => Effect.Effect<PreparedRequestOf<Body>, LLMError>
- readonly stream: StreamMethod
- readonly generate: GenerateMethod
- }
- export interface StreamMethod {
- (request: LLMRequest): Stream.Stream<LLMEvent, LLMError>
- }
- export interface GenerateMethod {
- (request: LLMRequest): Effect.Effect<LLMResponse, LLMError>
- }
- export class Service extends Context.Service<Service, Interface>()("@opencode/LLMClient") {}
- const resolveRequestOptions = (request: LLMRequest) => {
- const routeDefaults = request.model.route.defaults
- const modelDefaults = request.model.defaults
- const generation = mergeGenerationOptions(routeDefaults.generation, modelDefaults?.generation, request.generation)
- return LLMRequest.update(request, {
- generation: generation ?? new GenerationOptions({}),
- providerOptions: mergeProviderOptions(
- routeDefaults.providerOptions,
- modelDefaults?.providerOptions,
- request.providerOptions,
- ),
- http: mergeHttpOptions(routeDefaults.http, modelDefaults?.http, request.http),
- })
- }
- export interface MakeInput<Body, Frame, Event, State> {
- /** Route id used in diagnostics and prepared request metadata. */
- readonly id: string
- /** Provider identity for route-owned model construction. */
- readonly provider?: string | ProviderID
- /** Semantic API contract — owns body construction, body schema, and parsing. */
- readonly protocol: Protocol<Body, Frame, Event, State>
- /** Where the request is sent. */
- readonly endpoint: Endpoint<Body>
- /** Per-request transport auth. Provider facades override this via `route.with(...)`. */
- readonly auth?: AuthDef
- /** Stream framing — bytes -> frames before `protocol.stream.event` decoding. */
- readonly framing: Framing<Frame>
- /** Static / per-request headers added before `auth` runs. */
- readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
- /** Route/request defaults used when compiling requests for this route. */
- readonly defaults?: RouteDefaultsInput
- }
- export interface MakeTransportInput<Body, Prepared, Frame, Event, State> {
- /** Route id used in diagnostics and prepared request metadata. */
- readonly id: string
- /** Provider identity for route-owned model construction. */
- readonly provider?: string | ProviderID
- /** Semantic API contract — owns body construction, body schema, and parsing. */
- readonly protocol: Protocol<Body, Frame, Event, State>
- /** Where the request is sent. */
- readonly endpoint: Endpoint<Body>
- /** Per-request transport auth. Provider facades override this via `route.with(...)`. */
- readonly auth?: AuthDef
- /** Static / per-request headers added before `auth` runs. */
- readonly headers?: (input: { readonly request: LLMRequest }) => Record<string, string>
- /** Runnable transport route. */
- readonly transport: Transport<Body, Prepared, Frame>
- /** Route/request defaults used when compiling requests for this route. */
- readonly defaults?: RouteDefaultsInput
- }
- const streamError = (route: string, message: string, cause: Cause.Cause<unknown>) => {
- const failed = cause.reasons.find(Cause.isFailReason)?.error
- if (failed instanceof LLMErrorClass) return failed
- return ProviderShared.eventError(route, message, Cause.pretty(cause))
- }
- function makeFromTransport<Body, Prepared, Frame, Event, State>(
- input: MakeTransportInput<Body, Prepared, Frame, Event, State>,
- ): Route<Body, Prepared> {
- const protocol = input.protocol
- const encodeBody = Schema.encodeSync(Schema.fromJsonString(protocol.body.schema))
- const decodeEventEffect = Schema.decodeUnknownEffect(protocol.stream.event)
- const decodeEvent = (route: string) => (frame: Frame) =>
- decodeEventEffect(frame).pipe(
- Effect.mapError(() =>
- ProviderShared.eventError(
- input.id,
- `Invalid ${route} stream event`,
- typeof frame === "string" ? frame : ProviderShared.encodeJson(frame),
- ),
- ),
- )
- type BuiltRouteInput = Omit<MakeTransportInput<Body, Prepared, Frame, Event, State>, "defaults"> & {
- readonly defaults?: RouteDefaults
- }
- const build = (routeInput: BuiltRouteInput): Route<Body, Prepared> => {
- const route: Route<Body, Prepared> = {
- id: routeInput.id,
- provider: routeInput.provider === undefined ? undefined : ProviderID.make(routeInput.provider),
- protocol: protocol.id,
- endpoint: routeInput.endpoint,
- auth: routeInput.auth ?? Auth.none,
- transport: routeInput.transport,
- defaults: routeInput.defaults ?? {},
- body: protocol.body,
- with: (patch: RoutePatch<Body, Prepared>) => {
- const { id, provider, auth, transport, endpoint, ...defaults } = patch
- return build({
- ...routeInput,
- id: id ?? routeInput.id,
- provider: provider ?? routeInput.provider,
- auth: auth ?? routeInput.auth,
- endpoint: endpoint ? Endpoint.merge(routeInput.endpoint, endpoint) : routeInput.endpoint,
- transport: (transport as Transport<Body, Prepared, Frame> | undefined) ?? routeInput.transport,
- defaults: mergeRouteDefaults(route.defaults, defaults),
- })
- },
- model: (input) => makeRouteModel(route, input),
- prepareTransport: (body, request) =>
- routeInput.transport.prepare({
- body,
- request,
- endpoint: routeInput.endpoint,
- auth: routeInput.auth ?? Auth.none,
- encodeBody,
- headers: routeInput.headers,
- }),
- streamPrepared: (prepared: Prepared, request: LLMRequest, runtime: TransportRuntime) => {
- const route = `${request.model.provider}/${request.model.route.id}`
- const events = routeInput.transport
- .frames(prepared, request, runtime)
- .pipe(
- Stream.mapEffect(decodeEvent(route)),
- protocol.stream.terminal ? Stream.takeUntil(protocol.stream.terminal) : (stream) => stream,
- )
- return events.pipe(
- Stream.mapAccumEffect(
- () => protocol.stream.initial(request),
- protocol.stream.step,
- protocol.stream.onHalt ? { onHalt: protocol.stream.onHalt } : undefined,
- ),
- Stream.catchCause((cause) => Stream.fail(streamError(route, `Failed to read ${route} stream`, cause))),
- )
- },
- } satisfies Route<Body, Prepared>
- return route
- }
- return build({ ...input, defaults: mergeRouteDefaults(undefined, input.defaults ?? {}) })
- }
- export function make<Body, Prepared, Frame, Event, State>(
- input: MakeTransportInput<Body, Prepared, Frame, Event, State>,
- ): Route<Body, Prepared>
- /**
- * Build a `Route` by composing the four orthogonal pieces of a deployment:
- *
- * - `Protocol` — what is the API I'm speaking?
- * - `Endpoint` — where do I send the request?
- * - `Auth` — how do I authenticate it?
- * - `Framing` — how do I cut the response stream into protocol frames?
- *
- * Plus optional `headers` for cross-cutting deployment concerns (provider
- * version pins, per-deployment quirks).
- *
- * This is the canonical route constructor. If a new route does not fit
- * this four-axis model, add a purpose-built constructor rather than widening
- * the public surface preemptively.
- */
- export function make<Body, Frame, Event, State>(
- input: MakeInput<Body, Frame, Event, State>,
- ): Route<Body, HttpTransport.HttpPrepared<Frame>>
- export function make<Body, Prepared, Frame, Event, State>(
- input: MakeInput<Body, Frame, Event, State> | MakeTransportInput<Body, Prepared, Frame, Event, State>,
- ): Route<Body, Prepared> | Route<Body, HttpTransport.HttpPrepared<Frame>> {
- if ("transport" in input) return makeFromTransport(input)
- const protocol = input.protocol
- return makeFromTransport({
- id: input.id,
- provider: input.provider,
- protocol,
- endpoint: input.endpoint,
- auth: input.auth,
- headers: input.headers,
- transport: HttpTransport.httpJson({ framing: input.framing }),
- defaults: input.defaults,
- })
- }
- // `compile` is the important boundary: it turns a common `LLMRequest` into a
- // validated provider body plus transport-private prepared data, but does not
- // execute transport.
- const compile = Effect.fn("LLM.compile")(function* (request: LLMRequest) {
- const resolved = applyCachePolicy(resolveRequestOptions(request))
- const route = resolved.model.route
- const body = yield* route.body
- .from(resolved)
- .pipe(Effect.flatMap(ProviderShared.validateWith(Schema.decodeUnknownEffect(route.body.schema))))
- const prepared = yield* route.prepareTransport(body, resolved)
- return {
- request: resolved,
- route,
- body,
- prepared,
- }
- })
- const prepareWith = Effect.fn("LLMClient.prepare")(function* (request: LLMRequest) {
- const compiled = yield* compile(request)
- return new PreparedRequest({
- id: compiled.request.id ?? "request",
- route: compiled.route.id,
- protocol: compiled.route.protocol,
- model: compiled.request.model,
- body: compiled.body,
- metadata: { transport: compiled.route.transport.id },
- })
- })
- const streamRequestWith = (runtime: TransportRuntime) => (request: LLMRequest) =>
- Stream.unwrap(
- Effect.gen(function* () {
- const compiled = yield* compile(request)
- return compiled.route.streamPrepared(compiled.prepared, compiled.request, runtime)
- }),
- )
- const generateWith = (stream: Interface["stream"]) =>
- Effect.fn("LLM.generate")(function* (request: LLMRequest) {
- const state = yield* stream(request).pipe(Stream.runFold(LLMResponse.empty, LLMResponse.reduce))
- const response = LLMResponse.complete(state)
- if (response) return response
- return yield* ProviderShared.eventError(
- `${request.model.provider}/${request.model.route.id}`,
- "Provider stream ended without a terminal finish event",
- )
- })
- export const prepare = <Body = unknown>(request: LLMRequest) =>
- prepareWith(request) as Effect.Effect<PreparedRequestOf<Body>, LLMError>
- export function stream(request: LLMRequest): Stream.Stream<LLMEvent, LLMError> {
- return Stream.unwrap(
- Effect.gen(function* () {
- return (yield* Service).stream(request)
- }),
- ) as Stream.Stream<LLMEvent, LLMError>
- }
- export function generate(request: LLMRequest): Effect.Effect<LLMResponse, LLMError> {
- return Effect.gen(function* () {
- return yield* (yield* Service).generate(request)
- }) as Effect.Effect<LLMResponse, LLMError>
- }
- export const streamRequest = (request: LLMRequest) =>
- Stream.unwrap(
- Effect.gen(function* () {
- return (yield* Service).stream(request)
- }),
- )
- export const layer: Layer.Layer<Service, never, RequestExecutor.Service> = Layer.effect(
- Service,
- Effect.gen(function* () {
- const stream = streamRequestWith({
- http: yield* RequestExecutor.Service,
- webSocket: Option.getOrUndefined(yield* Effect.serviceOption(WebSocketExecutor.Service)),
- })
- return Service.of({ prepare: prepareWith as Interface["prepare"], stream, generate: generateWith(stream) })
- }),
- )
- export const Route = { make } as const
- export const LLMClient = {
- Service,
- layer,
- prepare,
- stream,
- generate,
- } as const
|