retry.ts 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199
  1. import type { NamedError } from "@opencode-ai/core/util/error"
  2. import { Cause, Clock, Duration, Effect, Schedule } from "effect"
  3. import { MessageV2 } from "./message-v2"
  4. import { iife } from "@/util/iife"
  5. export type Err = ReturnType<NamedError["toObject"]>
  6. export const GO_UPSELL_MESSAGE = "Free usage exceeded, subscribe to Go"
  7. export const GO_UPSELL_URL = "https://opencode.ai/go"
  8. export type RetryReason = "free_tier_limit" | "account_rate_limit" | (string & {})
  9. export type Retryable = {
  10. message: string
  11. action?: {
  12. reason: RetryReason
  13. provider: string
  14. title: string
  15. message: string
  16. label: string
  17. link?: string
  18. }
  19. }
  20. export const RETRY_INITIAL_DELAY = 2000
  21. export const RETRY_BACKOFF_FACTOR = 2
  22. export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds
  23. export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout
  24. function cap(ms: number) {
  25. return Math.min(ms, RETRY_MAX_DELAY)
  26. }
  27. export function delay(attempt: number, error?: MessageV2.APIError) {
  28. if (error) {
  29. const headers = error.data.responseHeaders
  30. if (headers) {
  31. const retryAfterMs = headers["retry-after-ms"]
  32. if (retryAfterMs) {
  33. const parsedMs = Number.parseFloat(retryAfterMs)
  34. if (!Number.isNaN(parsedMs)) {
  35. return cap(parsedMs)
  36. }
  37. }
  38. const retryAfter = headers["retry-after"]
  39. if (retryAfter) {
  40. const parsedSeconds = Number.parseFloat(retryAfter)
  41. if (!Number.isNaN(parsedSeconds)) {
  42. // convert seconds to milliseconds
  43. return cap(Math.ceil(parsedSeconds * 1000))
  44. }
  45. // Try parsing as HTTP date format
  46. const parsed = Date.parse(retryAfter) - Date.now()
  47. if (!Number.isNaN(parsed) && parsed > 0) {
  48. return cap(Math.ceil(parsed))
  49. }
  50. }
  51. return cap(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1))
  52. }
  53. }
  54. return cap(Math.min(RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), RETRY_MAX_DELAY_NO_HEADERS))
  55. }
  56. export function retryable(error: Err, provider: string) {
  57. // context overflow errors should not be retried
  58. if (MessageV2.ContextOverflowError.isInstance(error)) return undefined
  59. if (MessageV2.APIError.isInstance(error)) {
  60. const status = error.data.statusCode
  61. // 5xx errors are transient server failures and should always be retried,
  62. // even when the provider SDK doesn't explicitly mark them as retryable.
  63. if (!error.data.isRetryable && !(status !== undefined && status >= 500)) return undefined
  64. if (error.data.responseBody?.includes("FreeUsageLimitError")) {
  65. return {
  66. message: GO_UPSELL_MESSAGE,
  67. action: {
  68. reason: "free_tier_limit",
  69. provider,
  70. title: "Free limit reached",
  71. message: "Subscribe to OpenCode Go for reliable access to the best open-source models, starting at $5/month.",
  72. label: "subscribe",
  73. link: GO_UPSELL_URL,
  74. },
  75. }
  76. }
  77. if (error.data.responseBody?.includes("GoUsageLimitError")) {
  78. const body = parseJSON(error.data.responseBody)
  79. const workspace = str(body?.metadata?.workspace)
  80. const limitName = str(body?.metadata?.limitName)
  81. const retryAfter = num(error.data.responseHeaders?.["retry-after"])
  82. const resetIn = iife(() => {
  83. if (retryAfter === undefined) return ""
  84. const seconds = Math.max(0, Math.ceil(retryAfter))
  85. const days = Math.floor(seconds / 86_400)
  86. const hours = Math.floor((seconds % 86_400) / 3_600)
  87. const minutes = Math.ceil((seconds % 3_600) / 60)
  88. const unit = (value: number, name: string) => `${value} ${name}${value === 1 ? "" : "s"}`
  89. if (days > 0) return hours > 0 ? `${unit(days, "day")} ${unit(hours, "hour")}` : unit(days, "day")
  90. if (hours > 0) return minutes > 0 ? `${unit(hours, "hour")} ${unit(minutes, "minute")}` : unit(hours, "hour")
  91. return minutes > 0 ? unit(minutes, "minute") : "less than a minute"
  92. })
  93. const message = `${limitName ? `${limitName} usage limit` : "Usage limit"} reached. It will reset in ${resetIn}. To continue using this model now, enable usage from your available balance`
  94. const link = `https://opencode.ai/workspace/${workspace}/go`
  95. return {
  96. message: `${message} - ${link}`,
  97. action: {
  98. reason: "account_rate_limit",
  99. provider,
  100. title: "Go limit reached",
  101. message,
  102. label: "open settings",
  103. link,
  104. },
  105. }
  106. }
  107. return { message: error.data.message.includes("Overloaded") ? "Provider is overloaded" : error.data.message }
  108. }
  109. // Check for rate limit patterns in plain text error messages
  110. const msg = error.data?.message
  111. if (typeof msg === "string") {
  112. const lower = msg.toLowerCase()
  113. if (
  114. lower.includes("rate increased too quickly") ||
  115. lower.includes("rate limit") ||
  116. lower.includes("too many requests")
  117. ) {
  118. return { message: msg }
  119. }
  120. }
  121. const json = parseJSON(error.data?.message)
  122. if (!json || typeof json !== "object") return undefined
  123. const code = typeof json.code === "string" ? json.code : ""
  124. if (json.type === "error" && json.error?.type === "too_many_requests") {
  125. return { message: "Too Many Requests" }
  126. }
  127. if (code.includes("exhausted") || code.includes("unavailable")) {
  128. return { message: "Provider is overloaded" }
  129. }
  130. if (json.type === "error" && typeof json.error?.code === "string" && json.error.code.includes("rate_limit")) {
  131. return { message: "Rate Limited" }
  132. }
  133. return undefined
  134. }
  135. function str(value: unknown) {
  136. if (value === undefined || value === null) return ""
  137. return String(value)
  138. }
  139. function num(value: unknown) {
  140. const parsed = Number.parseFloat(str(value))
  141. if (Number.isNaN(parsed)) return undefined
  142. return parsed
  143. }
  144. function parseJSON(value: unknown) {
  145. return iife(() => {
  146. try {
  147. if (typeof value !== "string") return undefined
  148. return JSON.parse(value)
  149. } catch {
  150. return undefined
  151. }
  152. })
  153. }
  154. export function policy(opts: {
  155. provider: string
  156. parse: (error: unknown) => Err
  157. set: (input: { attempt: number; message: string; action?: Retryable["action"]; next: number }) => Effect.Effect<void>
  158. }) {
  159. return Schedule.fromStepWithMetadata(
  160. Effect.succeed((meta: Schedule.InputMetadata<unknown>) => {
  161. const error = opts.parse(meta.input)
  162. const retry = retryable(error, opts.provider)
  163. if (!retry) return Cause.done(meta.attempt)
  164. return Effect.gen(function* () {
  165. const wait = delay(meta.attempt, MessageV2.APIError.isInstance(error) ? error : undefined)
  166. const now = yield* Clock.currentTimeMillis
  167. yield* opts.set({
  168. attempt: meta.attempt,
  169. message: retry.message,
  170. action: retry.action,
  171. next: now + wait,
  172. })
  173. return [meta.attempt, Duration.millis(wait)] as [number, Duration.Duration]
  174. })
  175. }),
  176. )
  177. }
  178. export * as SessionRetry from "./retry"