feat(core): add flagged project references (#30414)

packages/core/src/config/reference.ts

@@ -15,3 +15,34 @@ export const Entry = Schema.Union([Schema.String, Git, Local])
 export type Entry = typeof Entry.Type
 
 export const Info = Schema.Record(Schema.String, Entry)
+export type Info = typeof Info.Type
+
+export type NormalizedEntry =
+  | { readonly kind: "local"; readonly path: string }
+  | { readonly kind: "git"; readonly repository: string; readonly branch?: string }
+  | { readonly kind: "invalid"; readonly message: string }
+
+export type NormalizedInfo = Record<string, NormalizedEntry>
+
+export function validateAlias(name: string) {
+  if (name.length === 0) return "Reference alias must not be empty"
+  if (/[\/\s`,]/.test(name)) return "Reference alias must not contain /, whitespace, comma, or backtick"
+}
+
+export function normalizeEntry(entry: Entry): NormalizedEntry {
+  if (typeof entry === "string") {
+    if (entry.startsWith(".") || entry.startsWith("/") || entry.startsWith("~")) return { kind: "local", path: entry }
+    return { kind: "git", repository: entry }
+  }
+  if ("path" in entry) return { kind: "local", path: entry.path }
+  return { kind: "git", repository: entry.repository, branch: entry.branch }
+}
+
+export function normalize(info: Info): NormalizedInfo {
+  return Object.fromEntries(
+    Object.entries(info).map(([name, entry]) => {
+      const message = validateAlias(name)
+      return [name, message ? { kind: "invalid" as const, message } : normalizeEntry(entry)]
+    }),
+  )
+}

packages/core/src/flag/flag.ts

@@ -5,11 +5,10 @@ function truthy(key: string) {
   return value === "true" || value === "1"
 }
 
-const OPENCODE_EXPERIMENTAL = truthy("OPENCODE_EXPERIMENTAL")
 const copy = process.env["OPENCODE_EXPERIMENTAL_DISABLE_COPY_ON_SELECT"]
 
 function enabledByExperimental(key: string) {
-  return process.env[key] === undefined ? OPENCODE_EXPERIMENTAL : truthy(key)
+  return process.env[key] === undefined ? truthy("OPENCODE_EXPERIMENTAL") : truthy(key)
 }
 
 export const Flag = {
@@ -54,6 +53,9 @@ export const Flag = {
   get OPENCODE_DISABLE_PROJECT_CONFIG() {
     return truthy("OPENCODE_DISABLE_PROJECT_CONFIG")
   },
+  get OPENCODE_EXPERIMENTAL_REFERENCES() {
+    return enabledByExperimental("OPENCODE_EXPERIMENTAL_REFERENCES")
+  },
   get OPENCODE_TUI_CONFIG() {
     return process.env["OPENCODE_TUI_CONFIG"]
   },

packages/core/src/location-layer.ts

@@ -18,6 +18,8 @@ import { PermissionV2 } from "./permission"
 import { PermissionSaved } from "./permission/saved"
 import { SessionV2 } from "./session"
 import { LocationFileSystem } from "./location-filesystem"
+import { ProjectReference } from "./project-reference"
+import { RepositoryCache } from "./repository-cache"
 
 export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("@opencode/example/LocationServiceMap", {
   lookup: (ref: Location.Ref) => {
@@ -26,6 +28,7 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
       location,
       Policy.locationLayer,
       Config.locationLayer,
+      ProjectReference.locationLayer,
       PluginV2.locationLayer,
       Catalog.locationLayer,
       AgentV2.locationLayer,
@@ -46,5 +49,6 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
     Database.defaultLayer,
     SessionV2.defaultLayer,
     PermissionSaved.defaultLayer,
+    RepositoryCache.defaultLayer,
   ],
 }) {}

packages/core/src/project-reference.ts

@@ -0,0 +1,208 @@
+export * as ProjectReference from "./project-reference"
+
+import path from "path"
+import { Context, Effect, Layer } from "effect"
+import { Config } from "./config"
+import { ConfigReference } from "./config/reference"
+import { AppFileSystem } from "./filesystem"
+import { Flag } from "./flag/flag"
+import { Global } from "./global"
+import { Location } from "./location"
+import { Repository } from "./repository"
+import { RepositoryCache } from "./repository-cache"
+
+export type Resolved =
+  | { readonly name: string; readonly kind: "local"; readonly path: string }
+  | {
+      readonly name: string
+      readonly kind: "git"
+      readonly repository: string
+      readonly reference: Repository.RemoteReference
+      readonly path: string
+      readonly branch?: string
+    }
+  | { readonly name: string; readonly kind: "invalid"; readonly repository?: string; readonly message: string }
+
+type Valid = Exclude<Resolved, { kind: "invalid" }>
+
+export type Mention =
+  | { readonly name: string; readonly kind: "reference"; readonly reference: Valid; readonly target?: string; readonly path: string }
+  | { readonly name: string; readonly kind: "invalid"; readonly target?: string; readonly message: string }
+  | { readonly name: string; readonly kind: "missing"; readonly target: string; readonly path: string; readonly message: string }
+
+export interface Interface {
+  readonly list: () => Effect.Effect<Resolved[]>
+  readonly get: (name: string) => Effect.Effect<Resolved | undefined>
+  readonly resolveMention: (value: string) => Effect.Effect<Mention | undefined, RepositoryCache.Error>
+  readonly ensurePath: (target?: string) => Effect.Effect<void, RepositoryCache.Error>
+  readonly containsManagedPath: (target?: string) => Effect.Effect<boolean>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@opencode/ProjectReference") {}
+
+type Materializer = {
+  readonly name: string
+  readonly repository: string
+  readonly path: string
+  readonly run: Effect.Effect<void, RepositoryCache.Error>
+}
+
+export const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    if (!Flag.OPENCODE_EXPERIMENTAL_REFERENCES) return Service.of(inert)
+
+    const config = yield* Config.Service
+    const fs = yield* AppFileSystem.Service
+    const global = yield* Global.Service
+    const location = yield* Location.Service
+    const cache = yield* RepositoryCache.Service
+    const references = resolveAll({
+      references: ConfigReference.normalize(
+        Object.assign({}, ...(yield* config.get()).map((document) => document.info.references ?? {})),
+      ),
+      directory: location.project.directory,
+      home: global.home,
+      repos: global.repos,
+    })
+    const materializers = yield* Effect.forEach(
+      uniqueGitReferences(references),
+      Effect.fnUntraced(function* (reference) {
+        return {
+          name: reference.name,
+          repository: reference.repository,
+          path: reference.path,
+          run: yield* Effect.cached(
+            cache.ensure({ reference: reference.reference, branch: reference.branch, refresh: true }).pipe(Effect.asVoid),
+          ),
+        }
+      }),
+    )
+
+    yield* Effect.forEach(
+      materializers,
+      (materializer) =>
+        materializer.run.pipe(
+          Effect.catchCause((cause) =>
+            Effect.logWarning("failed to materialize project reference").pipe(
+              Effect.annotateLogs({ name: materializer.name, repository: materializer.repository, cause }),
+            ),
+          ),
+        ),
+      { concurrency: 4, discard: true },
+    ).pipe(
+      Effect.forkScoped,
+    )
+
+    const ensurePath = Effect.fn("ProjectReference.ensurePath")(function* (target?: string) {
+      const normalized = normalizePath(target)
+      if (!normalized) return yield* Effect.forEach(materializers, (materializer) => materializer.run, { discard: true })
+      yield* materializers.find((materializer) => contains(materializer.path, normalized))?.run ?? Effect.void
+    })
+
+    return Service.of({
+      list: Effect.fn("ProjectReference.list")(function* () {
+        return references
+      }),
+      get: Effect.fn("ProjectReference.get")(function* (name: string) {
+        return references.find((reference) => reference.name === name)
+      }),
+      ensurePath,
+      containsManagedPath: Effect.fn("ProjectReference.containsManagedPath")(function* (target?: string) {
+        const normalized = normalizePath(target)
+        return normalized ? references.some((reference) => reference.kind === "git" && contains(reference.path, normalized)) : false
+      }),
+      resolveMention: Effect.fn("ProjectReference.resolveMention")(function* (value: string) {
+        const [name, ...rest] = value.split("/")
+        const target = rest.length ? rest.join("/") : undefined
+        const reference = references.find((reference) => reference.name === name)
+        if (!reference) return
+        if (reference.kind === "invalid") return { name, kind: "invalid", target, message: reference.message }
+        if (reference.kind === "git") yield* ensurePath(reference.path)
+        if (!target) return { name, kind: "reference", reference, path: reference.path }
+
+        const resolved = path.resolve(reference.path, target)
+        if (!AppFileSystem.contains(reference.path, resolved)) return { name, kind: "invalid", target, message: "Reference target escapes its root" }
+        if (!(yield* fs.existsSafe(resolved))) return { name, kind: "missing", target, path: resolved, message: "Reference target does not exist" }
+        return { name, kind: "reference", reference, target, path: resolved }
+      }),
+    })
+  }),
+)
+
+export const locationLayer = layer.pipe(Layer.provideMerge(Config.locationLayer))
+
+const inert: Interface = {
+  list: () => Effect.succeed([]),
+  get: () => Effect.succeed(undefined),
+  resolveMention: () => Effect.succeed(undefined),
+  ensurePath: () => Effect.void,
+  containsManagedPath: () => Effect.succeed(false),
+}
+
+export function resolveAll(input: { references: ConfigReference.NormalizedInfo; directory: string; home: string; repos: string }) {
+  const seen = new Map<string, { name: string; branch?: string }>()
+  return Object.entries(input.references).map(([name, reference]): Resolved => {
+    const resolved = resolve({ name, reference, directory: input.directory, home: input.home, repos: input.repos })
+    if (resolved.kind !== "git") return resolved
+    const existing = seen.get(resolved.path)
+    if (!existing) {
+      seen.set(resolved.path, { name, branch: resolved.branch })
+      return resolved
+    }
+    if (existing.branch === resolved.branch) return resolved
+    return {
+      name,
+      kind: "invalid",
+      repository: resolved.repository,
+      message: `Reference conflicts with @${existing.name}: both use ${resolved.path}, but @${existing.name} requests ${existing.branch ?? "default branch"} and @${name} requests ${resolved.branch ?? "default branch"}`,
+    }
+  })
+}
+
+export function resolve(input: { name: string; reference: ConfigReference.NormalizedEntry; directory: string; home: string; repos: string }): Resolved {
+  if (input.reference.kind === "invalid") return { name: input.name, kind: "invalid", message: input.reference.message }
+  if (input.reference.kind === "local") {
+    return { name: input.name, kind: "local", path: localPath(input.directory, input.home, input.reference.path) }
+  }
+  const reference = Repository.parse(input.reference.repository)
+  if (!reference || !Repository.isRemote(reference)) {
+    return {
+      name: input.name,
+      kind: "invalid",
+      repository: input.reference.repository,
+      message: "Repository must be a git URL, host/path reference, or GitHub owner/repo shorthand",
+    }
+  }
+  return {
+    name: input.name,
+    kind: "git",
+    repository: input.reference.repository,
+    reference,
+    path: Repository.cachePath(input.repos, reference),
+    branch: input.reference.branch,
+  }
+}
+
+function localPath(directory: string, home: string, value: string) {
+  if (value.startsWith("~/")) return path.join(home, value.slice(2))
+  return path.isAbsolute(value) ? value : path.resolve(directory, value)
+}
+
+function uniqueGitReferences(references: Resolved[]) {
+  const seen = new Set<string>()
+  return references.filter((reference): reference is Extract<Resolved, { kind: "git" }> => {
+    if (reference.kind !== "git" || seen.has(reference.path)) return false
+    seen.add(reference.path)
+    return true
+  })
+}
+
+function normalizePath(target?: string) {
+  if (!target) return
+  return process.platform === "win32" ? AppFileSystem.normalizePath(target) : target
+}
+
+function contains(parent: string, child: string) {
+  return AppFileSystem.contains(normalizePath(parent) ?? parent, normalizePath(child) ?? child)
+}

packages/core/test/location-layer.test.ts

@@ -16,6 +16,7 @@ import { Global } from "../src/global"
 import { ModelsDev } from "../src/models-dev"
 import { Npm } from "../src/npm"
 import { Project } from "../src/project"
+import { ProjectReference } from "../src/project-reference"
 
 const it = testEffect(
   LocationServiceMap.layer.pipe(
@@ -53,6 +54,7 @@ describe("LocationServiceMap", () => {
           const update = (directory: string) =>
             Effect.gen(function* () {
               yield* PluginBoot.Service.use((boot) => boot.wait())
+              yield* ProjectReference.Service
               const catalog = yield* Catalog.Service
               const transform = yield* catalog.transform()
               yield* transform((editor) => editor.provider.update(ProviderV2.ID.make("test"), () => {}))

packages/core/test/project-reference.test.ts

@@ -0,0 +1,267 @@
+import { describe, expect } from "bun:test"
+import fs from "fs/promises"
+import path from "path"
+import { Deferred, Effect, Layer, Schema } from "effect"
+import { Config } from "@opencode-ai/core/config"
+import { ConfigReference } from "@opencode-ai/core/config/reference"
+import { AppFileSystem } from "@opencode-ai/core/filesystem"
+import { Flag } from "@opencode-ai/core/flag/flag"
+import { Global } from "@opencode-ai/core/global"
+import { Location } from "@opencode-ai/core/location"
+import { ProjectReference } from "@opencode-ai/core/project-reference"
+import { Repository } from "@opencode-ai/core/repository"
+import { RepositoryCache } from "@opencode-ai/core/repository-cache"
+import { AbsolutePath } from "@opencode-ai/core/schema"
+import { location } from "./fixture/location"
+import { tmpdir } from "./fixture/tmpdir"
+import { it } from "./lib/effect"
+
+describe("ProjectReference", () => {
+  it.live("uses the broad experimental flag unless references are explicitly configured", () =>
+    withEnv({ OPENCODE_EXPERIMENTAL: "true", OPENCODE_EXPERIMENTAL_REFERENCES: undefined },
+      Effect.sync(() => {
+        expect(Flag.OPENCODE_EXPERIMENTAL_REFERENCES).toBe(true)
+      }),
+    ).pipe(
+      Effect.flatMap(() =>
+        withEnv({ OPENCODE_EXPERIMENTAL: "true", OPENCODE_EXPERIMENTAL_REFERENCES: "false" },
+          Effect.sync(() => {
+            expect(Flag.OPENCODE_EXPERIMENTAL_REFERENCES).toBe(false)
+          }),
+        ),
+      ),
+    ),
+  )
+
+  it.live("normalizes aliases and resolves relative local paths from the project root", () =>
+    withTmp((tmp) =>
+      Effect.gen(function* () {
+        const project = path.join(tmp.path, "project")
+        const nested = path.join(project, "packages", "app")
+        yield* Effect.promise(() => fs.mkdir(nested, { recursive: true }))
+
+        const references = ProjectReference.resolveAll({
+          references: ConfigReference.normalize({
+            docs: { path: "./docs" },
+            home: "~/notes",
+            sdk: { repository: "owner/repo", branch: "main" },
+            shorthand: "owner/other",
+            invalid: "not-a-repo",
+            "bad/name": "owner/repo",
+          }),
+          directory: project,
+          home: path.join(tmp.path, "home"),
+          repos: path.join(tmp.path, "repos"),
+        })
+
+        expect(references).toMatchObject([
+          { name: "docs", kind: "local", path: path.join(project, "docs") },
+          { name: "home", kind: "local", path: path.join(tmp.path, "home", "notes") },
+          { name: "sdk", kind: "git", branch: "main" },
+          { name: "shorthand", kind: "git" },
+          { name: "invalid", kind: "invalid", repository: "not-a-repo" },
+          { name: "bad/name", kind: "invalid" },
+        ])
+      }),
+    ),
+  )
+
+  it.live("marks same-cache references with different branches invalid", () =>
+    Effect.sync(() => {
+      const references = ProjectReference.resolveAll({
+        references: ConfigReference.normalize({
+          main: { repository: "owner/repo", branch: "main" },
+          dev: { repository: "github.com/owner/repo", branch: "dev" },
+          alsoMain: { repository: "https://github.com/owner/repo", branch: "main" },
+        }),
+        directory: "/project",
+        home: "/home",
+        repos: "/repos",
+      })
+
+      expect(references.map((reference) => reference.kind)).toEqual(["git", "invalid", "git"])
+      expect(references[1]?.kind === "invalid" ? references[1].message : "").toContain("conflicts with @main")
+    }),
+  )
+
+  it.live("merges config aliases and exposes mention and managed-path operations", () =>
+    withoutReferences(withTmp((tmp) => {
+      const calls: RepositoryCache.EnsureInput[] = []
+      const project = path.join(tmp.path, "project")
+      const nested = path.join(project, "packages", "app")
+      const docs = path.join(project, "docs")
+      const repos = path.join(tmp.path, "repos")
+      return Effect.gen(function* () {
+        yield* Effect.promise(async () => {
+          await fs.mkdir(nested, { recursive: true })
+          await fs.mkdir(docs)
+          await fs.writeFile(path.join(docs, "README.md"), "docs")
+        })
+
+        yield* withReferences(
+          Effect.gen(function* () {
+            const references = yield* ProjectReference.Service
+            const git = path.join(repos, "github.com", "owner", "repo")
+
+            expect(yield* references.list()).toMatchObject([
+              { name: "docs", kind: "local", path: docs },
+              { name: "sdk", kind: "git", path: git },
+            ])
+            expect(yield* references.resolveMention("docs/README.md")).toMatchObject({
+              name: "docs",
+              kind: "reference",
+              target: "README.md",
+              path: path.join(docs, "README.md"),
+            })
+            expect(yield* references.resolveMention("docs/missing.md")).toMatchObject({ name: "docs", kind: "missing" })
+            expect(yield* references.resolveMention("docs/../outside.md")).toMatchObject({ name: "docs", kind: "invalid" })
+            expect(yield* references.resolveMention("unknown")).toBeUndefined()
+            expect(yield* references.resolveMention("sdk")).toMatchObject({ name: "sdk", kind: "reference", path: git })
+            expect(yield* references.containsManagedPath(path.join(git, "README.md"))).toBe(true)
+            expect(yield* references.containsManagedPath(path.join(docs, "README.md"))).toBe(false)
+            yield* references.ensurePath()
+            expect(calls).toHaveLength(1)
+          }).pipe(
+            Effect.provide(
+              testLayer({
+                directory: nested,
+                project,
+                repos,
+                documents: [
+                  document({ docs: { path: "./old-docs" }, sdk: "owner/old" }),
+                  document({ docs: { path: "./docs" }, sdk: { repository: "owner/repo", branch: "main" } }),
+                ],
+                ensure: (input) => Effect.sync(() => result(repos, calls, input)),
+              }),
+            ),
+          ),
+        )
+      })
+    })),
+  )
+
+  it.live("is inert while the runtime flag is disabled", () =>
+    withoutReferences(withTmp((tmp) => {
+      const calls: RepositoryCache.EnsureInput[] = []
+      return Effect.gen(function* () {
+        const references = yield* ProjectReference.Service
+        expect(yield* references.list()).toEqual([])
+        expect(yield* references.get("sdk")).toBeUndefined()
+        expect(yield* references.resolveMention("sdk")).toBeUndefined()
+        expect(yield* references.containsManagedPath(path.join(tmp.path, "repos", "github.com", "owner", "repo"))).toBe(false)
+        yield* references.ensurePath()
+        expect(calls).toEqual([])
+      }).pipe(
+        Effect.provide(
+          testLayer({
+            directory: tmp.path,
+            project: tmp.path,
+            repos: path.join(tmp.path, "repos"),
+            documents: [document({ sdk: "owner/repo" })],
+            ensure: (input) => Effect.sync(() => result(path.join(tmp.path, "repos"), calls, input)),
+          }),
+        ),
+      )
+    })),
+  )
+
+  it.live("starts Git materialization in the background without blocking the location layer", () =>
+    withTmp((tmp) =>
+      Effect.gen(function* () {
+        const started = yield* Deferred.make<void>()
+        yield* withReferences(
+          Effect.gen(function* () {
+            expect(yield* (yield* ProjectReference.Service).list()).toHaveLength(1)
+            yield* Deferred.await(started).pipe(
+              Effect.timeoutOrElse({ duration: "1 second", orElse: () => Effect.die(new Error("refresh did not start")) }),
+            )
+          }).pipe(
+            Effect.provide(
+              testLayer({
+                directory: tmp.path,
+                project: tmp.path,
+                repos: path.join(tmp.path, "repos"),
+                documents: [document({ sdk: "owner/repo" })],
+                ensure: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)),
+              }),
+            ),
+          ),
+        )
+      }),
+    ),
+  )
+})
+
+function document(references: ConfigReference.Info) {
+  return new Config.Loaded({ source: { type: "memory" }, info: Schema.decodeUnknownSync(Config.Info)({ references }) })
+}
+
+function result(repos: string, calls: RepositoryCache.EnsureInput[], input: RepositoryCache.EnsureInput): RepositoryCache.Result {
+  calls.push(input)
+  return {
+    repository: input.reference.label,
+    host: input.reference.host,
+    remote: input.reference.remote,
+    localPath: Repository.cachePath(repos, input.reference),
+    status: "cached",
+    branch: input.branch,
+  }
+}
+
+function testLayer(input: {
+  directory: string
+  project: string
+  repos: string
+  documents: Config.Loaded[]
+  ensure: RepositoryCache.Interface["ensure"]
+}) {
+  return ProjectReference.layer.pipe(
+    Layer.provide(
+      Layer.mergeAll(
+        AppFileSystem.defaultLayer,
+        Global.layerWith({ home: path.join(input.directory, "home"), repos: input.repos }),
+        Layer.succeed(
+          Location.Service,
+          Location.Service.of(
+            location({ directory: AbsolutePath.make(input.directory) }, { projectDirectory: AbsolutePath.make(input.project) }),
+          ),
+        ),
+        Layer.succeed(Config.Service, Config.Service.of({ directories: () => Effect.succeed([]), get: () => Effect.succeed(input.documents) })),
+        Layer.succeed(RepositoryCache.Service, RepositoryCache.Service.of({ ensure: input.ensure })),
+      ),
+    ),
+  )
+}
+
+function withTmp<A, E, R>(body: (tmp: Awaited<ReturnType<typeof tmpdir>>) => Effect.Effect<A, E, R>) {
+  return Effect.acquireUseRelease(Effect.promise(() => tmpdir()), body, (tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]()))
+}
+
+function withReferences<A, E, R>(body: Effect.Effect<A, E, R>) {
+  return withEnv({ OPENCODE_EXPERIMENTAL_REFERENCES: "true" }, body)
+}
+
+function withoutReferences<A, E, R>(body: Effect.Effect<A, E, R>) {
+  return withEnv({ OPENCODE_EXPERIMENTAL: undefined, OPENCODE_EXPERIMENTAL_REFERENCES: undefined }, body)
+}
+
+function withEnv<A, E, R>(env: Record<string, string | undefined>, body: Effect.Effect<A, E, R>) {
+  return Effect.acquireUseRelease(
+    Effect.sync(() => {
+      const previous = Object.fromEntries(Object.keys(env).map((key) => [key, process.env[key]]))
+      for (const [key, value] of Object.entries(env)) {
+        if (value === undefined) delete process.env[key]
+        else process.env[key] = value
+      }
+      return previous
+    }),
+    () => body,
+    (previous) =>
+      Effect.sync(() => {
+        for (const [key, value] of Object.entries(previous)) {
+          if (value === undefined) delete process.env[key]
+          else process.env[key] = value
+        }
+      }),
+  )
+}