index.ts 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583
  1. import { BusEvent } from "@/bus/bus-event"
  2. import z from "zod"
  3. import { $ } from "bun"
  4. import type { BunFile } from "bun"
  5. import { formatPatch, structuredPatch } from "diff"
  6. import path from "path"
  7. import fs from "fs"
  8. import ignore from "ignore"
  9. import { Log } from "../util/log"
  10. import { Filesystem } from "../util/filesystem"
  11. import { Instance } from "../project/instance"
  12. import { Ripgrep } from "./ripgrep"
  13. import fuzzysort from "fuzzysort"
  14. import { Global } from "../global"
  15. export namespace File {
  16. const log = Log.create({ service: "file" })
  17. export const Info = z
  18. .object({
  19. path: z.string(),
  20. added: z.number().int(),
  21. removed: z.number().int(),
  22. status: z.enum(["added", "deleted", "modified"]),
  23. })
  24. .meta({
  25. ref: "File",
  26. })
  27. export type Info = z.infer<typeof Info>
  28. export const Node = z
  29. .object({
  30. name: z.string(),
  31. path: z.string(),
  32. absolute: z.string(),
  33. type: z.enum(["file", "directory"]),
  34. ignored: z.boolean(),
  35. })
  36. .meta({
  37. ref: "FileNode",
  38. })
  39. export type Node = z.infer<typeof Node>
  40. export const Content = z
  41. .object({
  42. type: z.enum(["text", "binary"]),
  43. content: z.string(),
  44. diff: z.string().optional(),
  45. patch: z
  46. .object({
  47. oldFileName: z.string(),
  48. newFileName: z.string(),
  49. oldHeader: z.string().optional(),
  50. newHeader: z.string().optional(),
  51. hunks: z.array(
  52. z.object({
  53. oldStart: z.number(),
  54. oldLines: z.number(),
  55. newStart: z.number(),
  56. newLines: z.number(),
  57. lines: z.array(z.string()),
  58. }),
  59. ),
  60. index: z.string().optional(),
  61. })
  62. .optional(),
  63. encoding: z.literal("base64").optional(),
  64. mimeType: z.string().optional(),
  65. })
  66. .meta({
  67. ref: "FileContent",
  68. })
  69. export type Content = z.infer<typeof Content>
  70. const binaryExtensions = new Set([
  71. "exe",
  72. "dll",
  73. "pdb",
  74. "bin",
  75. "so",
  76. "dylib",
  77. "o",
  78. "a",
  79. "lib",
  80. "wav",
  81. "mp3",
  82. "ogg",
  83. "oga",
  84. "ogv",
  85. "ogx",
  86. "flac",
  87. "aac",
  88. "wma",
  89. "m4a",
  90. "weba",
  91. "mp4",
  92. "avi",
  93. "mov",
  94. "wmv",
  95. "flv",
  96. "webm",
  97. "mkv",
  98. "zip",
  99. "tar",
  100. "gz",
  101. "gzip",
  102. "bz",
  103. "bz2",
  104. "bzip",
  105. "bzip2",
  106. "7z",
  107. "rar",
  108. "xz",
  109. "lz",
  110. "z",
  111. "pdf",
  112. "doc",
  113. "docx",
  114. "ppt",
  115. "pptx",
  116. "xls",
  117. "xlsx",
  118. "dmg",
  119. "iso",
  120. "img",
  121. "vmdk",
  122. "ttf",
  123. "otf",
  124. "woff",
  125. "woff2",
  126. "eot",
  127. "sqlite",
  128. "db",
  129. "mdb",
  130. "apk",
  131. "ipa",
  132. "aab",
  133. "xapk",
  134. "app",
  135. "pkg",
  136. "deb",
  137. "rpm",
  138. "snap",
  139. "flatpak",
  140. "appimage",
  141. "msi",
  142. "msp",
  143. "jar",
  144. "war",
  145. "ear",
  146. "class",
  147. "kotlin_module",
  148. "dex",
  149. "vdex",
  150. "odex",
  151. "oat",
  152. "art",
  153. "wasm",
  154. "wat",
  155. "bc",
  156. "ll",
  157. "s",
  158. "ko",
  159. "sys",
  160. "drv",
  161. "efi",
  162. "rom",
  163. "com",
  164. "bat",
  165. "cmd",
  166. "ps1",
  167. "sh",
  168. "bash",
  169. "zsh",
  170. "fish",
  171. ])
  172. const imageExtensions = new Set([
  173. "png",
  174. "jpg",
  175. "jpeg",
  176. "gif",
  177. "bmp",
  178. "webp",
  179. "ico",
  180. "tif",
  181. "tiff",
  182. "svg",
  183. "svgz",
  184. "avif",
  185. "apng",
  186. "jxl",
  187. "heic",
  188. "heif",
  189. "raw",
  190. "cr2",
  191. "nef",
  192. "arw",
  193. "dng",
  194. "orf",
  195. "raf",
  196. "pef",
  197. "x3f",
  198. ])
  199. function isImageByExtension(filepath: string): boolean {
  200. const ext = path.extname(filepath).toLowerCase().slice(1)
  201. return imageExtensions.has(ext)
  202. }
  203. function getImageMimeType(filepath: string): string {
  204. const ext = path.extname(filepath).toLowerCase().slice(1)
  205. const mimeTypes: Record<string, string> = {
  206. png: "image/png",
  207. jpg: "image/jpeg",
  208. jpeg: "image/jpeg",
  209. gif: "image/gif",
  210. bmp: "image/bmp",
  211. webp: "image/webp",
  212. ico: "image/x-icon",
  213. tif: "image/tiff",
  214. tiff: "image/tiff",
  215. svg: "image/svg+xml",
  216. svgz: "image/svg+xml",
  217. avif: "image/avif",
  218. apng: "image/apng",
  219. jxl: "image/jxl",
  220. heic: "image/heic",
  221. heif: "image/heif",
  222. }
  223. return mimeTypes[ext] || "image/" + ext
  224. }
  225. function isBinaryByExtension(filepath: string): boolean {
  226. const ext = path.extname(filepath).toLowerCase().slice(1)
  227. return binaryExtensions.has(ext)
  228. }
  229. function isImage(mimeType: string): boolean {
  230. return mimeType.startsWith("image/")
  231. }
  232. async function shouldEncode(file: BunFile): Promise<boolean> {
  233. const type = file.type?.toLowerCase()
  234. log.info("shouldEncode", { type })
  235. if (!type) return false
  236. if (type.startsWith("text/")) return false
  237. if (type.includes("charset=")) return false
  238. const parts = type.split("/", 2)
  239. const top = parts[0]
  240. const tops = ["image", "audio", "video", "font", "model", "multipart"]
  241. if (tops.includes(top)) return true
  242. return false
  243. }
  244. export const Event = {
  245. Edited: BusEvent.define(
  246. "file.edited",
  247. z.object({
  248. file: z.string(),
  249. }),
  250. ),
  251. }
  252. const state = Instance.state(async () => {
  253. type Entry = { files: string[]; dirs: string[] }
  254. let cache: Entry = { files: [], dirs: [] }
  255. let fetching = false
  256. const isGlobalHome = Instance.directory === Global.Path.home && Instance.project.id === "global"
  257. const fn = async (result: Entry) => {
  258. // Disable scanning if in root of file system
  259. if (Instance.directory === path.parse(Instance.directory).root) return
  260. fetching = true
  261. if (isGlobalHome) {
  262. const dirs = new Set<string>()
  263. const ignore = new Set<string>()
  264. if (process.platform === "darwin") ignore.add("Library")
  265. if (process.platform === "win32") ignore.add("AppData")
  266. const ignoreNested = new Set(["node_modules", "dist", "build", "target", "vendor"])
  267. const shouldIgnore = (name: string) => name.startsWith(".") || ignore.has(name)
  268. const shouldIgnoreNested = (name: string) => name.startsWith(".") || ignoreNested.has(name)
  269. const top = await fs.promises
  270. .readdir(Instance.directory, { withFileTypes: true })
  271. .catch(() => [] as fs.Dirent[])
  272. for (const entry of top) {
  273. if (!entry.isDirectory()) continue
  274. if (shouldIgnore(entry.name)) continue
  275. dirs.add(entry.name + "/")
  276. const base = path.join(Instance.directory, entry.name)
  277. const children = await fs.promises.readdir(base, { withFileTypes: true }).catch(() => [] as fs.Dirent[])
  278. for (const child of children) {
  279. if (!child.isDirectory()) continue
  280. if (shouldIgnoreNested(child.name)) continue
  281. dirs.add(entry.name + "/" + child.name + "/")
  282. }
  283. }
  284. result.dirs = Array.from(dirs).toSorted()
  285. cache = result
  286. fetching = false
  287. return
  288. }
  289. const set = new Set<string>()
  290. for await (const file of Ripgrep.files({ cwd: Instance.directory })) {
  291. result.files.push(file)
  292. let current = file
  293. while (true) {
  294. const dir = path.dirname(current)
  295. if (dir === ".") break
  296. if (dir === current) break
  297. current = dir
  298. if (set.has(dir)) continue
  299. set.add(dir)
  300. result.dirs.push(dir + "/")
  301. }
  302. }
  303. cache = result
  304. fetching = false
  305. }
  306. fn(cache)
  307. return {
  308. async files() {
  309. if (!fetching) {
  310. fn({
  311. files: [],
  312. dirs: [],
  313. })
  314. }
  315. return cache
  316. },
  317. }
  318. })
  319. export function init() {
  320. state()
  321. }
  322. export async function status() {
  323. const project = Instance.project
  324. if (project.vcs !== "git") return []
  325. const diffOutput = await $`git -c core.quotepath=false diff --numstat HEAD`
  326. .cwd(Instance.directory)
  327. .quiet()
  328. .nothrow()
  329. .text()
  330. const changedFiles: Info[] = []
  331. if (diffOutput.trim()) {
  332. const lines = diffOutput.trim().split("\n")
  333. for (const line of lines) {
  334. const [added, removed, filepath] = line.split("\t")
  335. changedFiles.push({
  336. path: filepath,
  337. added: added === "-" ? 0 : parseInt(added, 10),
  338. removed: removed === "-" ? 0 : parseInt(removed, 10),
  339. status: "modified",
  340. })
  341. }
  342. }
  343. const untrackedOutput = await $`git -c core.quotepath=false ls-files --others --exclude-standard`
  344. .cwd(Instance.directory)
  345. .quiet()
  346. .nothrow()
  347. .text()
  348. if (untrackedOutput.trim()) {
  349. const untrackedFiles = untrackedOutput.trim().split("\n")
  350. for (const filepath of untrackedFiles) {
  351. try {
  352. const content = await Bun.file(path.join(Instance.directory, filepath)).text()
  353. const lines = content.split("\n").length
  354. changedFiles.push({
  355. path: filepath,
  356. added: lines,
  357. removed: 0,
  358. status: "added",
  359. })
  360. } catch {
  361. continue
  362. }
  363. }
  364. }
  365. // Get deleted files
  366. const deletedOutput = await $`git -c core.quotepath=false diff --name-only --diff-filter=D HEAD`
  367. .cwd(Instance.directory)
  368. .quiet()
  369. .nothrow()
  370. .text()
  371. if (deletedOutput.trim()) {
  372. const deletedFiles = deletedOutput.trim().split("\n")
  373. for (const filepath of deletedFiles) {
  374. changedFiles.push({
  375. path: filepath,
  376. added: 0,
  377. removed: 0, // Could get original line count but would require another git command
  378. status: "deleted",
  379. })
  380. }
  381. }
  382. return changedFiles.map((x) => ({
  383. ...x,
  384. path: path.relative(Instance.directory, x.path),
  385. }))
  386. }
  387. export async function read(file: string): Promise<Content> {
  388. using _ = log.time("read", { file })
  389. const project = Instance.project
  390. const full = path.join(Instance.directory, file)
  391. // TODO: Filesystem.contains is lexical only - symlinks inside the project can escape.
  392. // TODO: On Windows, cross-drive paths bypass this check. Consider realpath canonicalization.
  393. if (!Instance.containsPath(full)) {
  394. throw new Error(`Access denied: path escapes project directory`)
  395. }
  396. // Fast path: check extension before any filesystem operations
  397. if (isImageByExtension(file)) {
  398. const bunFile = Bun.file(full)
  399. if (await bunFile.exists()) {
  400. const buffer = await bunFile.arrayBuffer().catch(() => new ArrayBuffer(0))
  401. const content = Buffer.from(buffer).toString("base64")
  402. const mimeType = getImageMimeType(file)
  403. return { type: "text", content, mimeType, encoding: "base64" }
  404. }
  405. return { type: "text", content: "" }
  406. }
  407. if (isBinaryByExtension(file)) {
  408. return { type: "binary", content: "" }
  409. }
  410. const bunFile = Bun.file(full)
  411. if (!(await bunFile.exists())) {
  412. return { type: "text", content: "" }
  413. }
  414. const encode = await shouldEncode(bunFile)
  415. const mimeType = bunFile.type || "application/octet-stream"
  416. if (encode && !isImage(mimeType)) {
  417. return { type: "binary", content: "", mimeType }
  418. }
  419. if (encode) {
  420. const buffer = await bunFile.arrayBuffer().catch(() => new ArrayBuffer(0))
  421. const content = Buffer.from(buffer).toString("base64")
  422. return { type: "text", content, mimeType, encoding: "base64" }
  423. }
  424. const content = await bunFile
  425. .text()
  426. .catch(() => "")
  427. .then((x) => x.trim())
  428. if (project.vcs === "git") {
  429. let diff = await $`git diff ${file}`.cwd(Instance.directory).quiet().nothrow().text()
  430. if (!diff.trim()) diff = await $`git diff --staged ${file}`.cwd(Instance.directory).quiet().nothrow().text()
  431. if (diff.trim()) {
  432. const original = await $`git show HEAD:${file}`.cwd(Instance.directory).quiet().nothrow().text()
  433. const patch = structuredPatch(file, file, original, content, "old", "new", {
  434. context: Infinity,
  435. ignoreWhitespace: true,
  436. })
  437. const diff = formatPatch(patch)
  438. return { type: "text", content, patch, diff }
  439. }
  440. }
  441. return { type: "text", content }
  442. }
  443. export async function list(dir?: string) {
  444. const exclude = [".git", ".DS_Store"]
  445. const project = Instance.project
  446. let ignored = (_: string) => false
  447. if (project.vcs === "git") {
  448. const ig = ignore()
  449. const gitignore = Bun.file(path.join(Instance.worktree, ".gitignore"))
  450. if (await gitignore.exists()) {
  451. ig.add(await gitignore.text())
  452. }
  453. const ignoreFile = Bun.file(path.join(Instance.worktree, ".ignore"))
  454. if (await ignoreFile.exists()) {
  455. ig.add(await ignoreFile.text())
  456. }
  457. ignored = ig.ignores.bind(ig)
  458. }
  459. const resolved = dir ? path.join(Instance.directory, dir) : Instance.directory
  460. // TODO: Filesystem.contains is lexical only - symlinks inside the project can escape.
  461. // TODO: On Windows, cross-drive paths bypass this check. Consider realpath canonicalization.
  462. if (!Instance.containsPath(resolved)) {
  463. throw new Error(`Access denied: path escapes project directory`)
  464. }
  465. const nodes: Node[] = []
  466. for (const entry of await fs.promises
  467. .readdir(resolved, {
  468. withFileTypes: true,
  469. })
  470. .catch(() => [])) {
  471. if (exclude.includes(entry.name)) continue
  472. const fullPath = path.join(resolved, entry.name)
  473. const relativePath = path.relative(Instance.directory, fullPath)
  474. const type = entry.isDirectory() ? "directory" : "file"
  475. nodes.push({
  476. name: entry.name,
  477. path: relativePath,
  478. absolute: fullPath,
  479. type,
  480. ignored: ignored(type === "directory" ? relativePath + "/" : relativePath),
  481. })
  482. }
  483. return nodes.sort((a, b) => {
  484. if (a.type !== b.type) {
  485. return a.type === "directory" ? -1 : 1
  486. }
  487. return a.name.localeCompare(b.name)
  488. })
  489. }
  490. export async function search(input: { query: string; limit?: number; dirs?: boolean; type?: "file" | "directory" }) {
  491. const query = input.query.trim()
  492. const limit = input.limit ?? 100
  493. const kind = input.type ?? (input.dirs === false ? "file" : "all")
  494. log.info("search", { query, kind })
  495. const result = await state().then((x) => x.files())
  496. const hidden = (item: string) => {
  497. const normalized = item.replaceAll("\\", "/").replace(/\/+$/, "")
  498. return normalized.split("/").some((p) => p.startsWith(".") && p.length > 1)
  499. }
  500. const preferHidden = query.startsWith(".") || query.includes("/.")
  501. const sortHiddenLast = (items: string[]) => {
  502. if (preferHidden) return items
  503. const visible: string[] = []
  504. const hiddenItems: string[] = []
  505. for (const item of items) {
  506. const isHidden = hidden(item)
  507. if (isHidden) hiddenItems.push(item)
  508. if (!isHidden) visible.push(item)
  509. }
  510. return [...visible, ...hiddenItems]
  511. }
  512. if (!query) {
  513. if (kind === "file") return result.files.slice(0, limit)
  514. return sortHiddenLast(result.dirs.toSorted()).slice(0, limit)
  515. }
  516. const items =
  517. kind === "file" ? result.files : kind === "directory" ? result.dirs : [...result.files, ...result.dirs]
  518. const searchLimit = kind === "directory" && !preferHidden ? limit * 20 : limit
  519. const sorted = fuzzysort.go(query, items, { limit: searchLimit }).map((r) => r.target)
  520. const output = kind === "directory" ? sortHiddenLast(sorted).slice(0, limit) : sorted
  521. log.info("search", { query, kind, results: output.length })
  522. return output
  523. }
  524. }