mock-tools.ts 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330
  1. import assert from 'node:assert'
  2. import { tool } from 'ai'
  3. import { z } from 'zod'
  4. import { getStudioTools } from '../tools/studio-tools'
  5. import { getMcpTools } from '@/lib/ai/tools/mcp-tools'
  6. const listTablesInputSchema = z.object({
  7. schemas: z.array(z.string()).describe('The schema names to list.'),
  8. })
  9. const getAdvisorsInputSchema = z.object({
  10. type: z.enum(['security', 'performance']).optional(),
  11. })
  12. const getLogsInputSchema = z.object({
  13. limit: z.number().min(1).max(100).optional(),
  14. level: z.enum(['debug', 'info', 'warning', 'error']).optional(),
  15. source: z.enum(['postgres', 'auth', 'storage', 'edge_function']).optional(),
  16. search: z.string().optional(),
  17. })
  18. const listPoliciesInputSchema = z.object({
  19. schemas: z.array(z.string()).describe('The schema names to get the policies for'),
  20. })
  21. export const MOCK_TABLES_DATA = [
  22. {
  23. name: 'user_documents',
  24. rls_enabled: false,
  25. columns: [
  26. { name: 'id', data_type: 'bigint' },
  27. { name: 'user_id', data_type: 'uuid' },
  28. { name: 'title', data_type: 'text' },
  29. ],
  30. },
  31. {
  32. name: 'customers',
  33. rls_enabled: true,
  34. columns: [
  35. { name: 'id', data_type: 'uuid' },
  36. { name: 'tenant_id', data_type: 'uuid' },
  37. { name: 'email', data_type: 'text' },
  38. ],
  39. },
  40. {
  41. name: 'projects',
  42. rls_enabled: false,
  43. columns: [
  44. { name: 'id', data_type: 'uuid' },
  45. { name: 'organization_id', data_type: 'uuid' },
  46. { name: 'name', data_type: 'text' },
  47. ],
  48. },
  49. {
  50. name: 'user_organizations',
  51. rls_enabled: true,
  52. columns: [
  53. { name: 'user_id', data_type: 'uuid' },
  54. { name: 'organization_id', data_type: 'uuid' },
  55. ],
  56. },
  57. ]
  58. const MOCK_EXTENSIONS_DATA = [
  59. { name: 'pgcrypto', schema: 'extensions', installed_version: '1.3' },
  60. { name: 'uuid-ossp', schema: 'extensions', installed_version: '1.1' },
  61. { name: 'pg_cron', schema: 'pg_catalog', installed_version: '1.6.4' },
  62. ]
  63. const MOCK_EDGE_FUNCTIONS_DATA = [
  64. { name: 'hello-world', last_deployed_at: '2024-06-10T12:30:00Z' },
  65. { name: 'daily-metrics-sync', last_deployed_at: '2024-06-18T08:15:00Z' },
  66. { name: 'select-from-table-with-auth-rls', last_deployed_at: '2024-06-19T09:20:00Z' },
  67. ]
  68. const MOCK_ADVISORIES_DATA = [
  69. {
  70. id: '0016_materialized_view_in_api',
  71. level: 'warning',
  72. category: 'security',
  73. message: 'Materialized views in API schema can bypass RLS. Move them to private schema.',
  74. remediationUrl:
  75. 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0016_materialized_view_in_api',
  76. },
  77. {
  78. id: '0031_functions_no_rls_guard',
  79. level: 'notice',
  80. category: 'security',
  81. message: 'Function api.health_check should verify auth context before querying tables.',
  82. remediationUrl:
  83. 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0031_functions_no_rls_guard',
  84. },
  85. {
  86. id: '1012_slow_query',
  87. level: 'info',
  88. category: 'performance',
  89. message:
  90. 'Query on table edge_function_logs exceeded 3s average execution time over the last hour.',
  91. remediationUrl: 'https://supabase.com/docs/guides/platform/performance-advisors#slow-queries',
  92. },
  93. ]
  94. const MOCK_LOGS_DATA = [
  95. {
  96. id: 'log-001',
  97. timestamp: '2024-06-20T14:12:00Z',
  98. level: 'error',
  99. source: 'edge_function' as const,
  100. target: 'hello-world',
  101. message: "TypeError: fetch failed at await briven.functions.invoke('analytics')",
  102. },
  103. {
  104. id: 'log-002',
  105. timestamp: '2024-06-20T14:05:30Z',
  106. level: 'warning',
  107. source: 'postgres' as const,
  108. target: 'connection_pool',
  109. message: 'Query timeout exceeded for statement SELECT * FROM public.audit_log_entries',
  110. },
  111. {
  112. id: 'log-003',
  113. timestamp: '2024-06-20T13:59:10Z',
  114. level: 'info',
  115. source: 'edge_function' as const,
  116. target: 'daily-metrics-sync',
  117. message: 'Invocation completed in 520ms',
  118. },
  119. {
  120. id: 'log-004',
  121. timestamp: '2024-06-20T13:50:00Z',
  122. level: 'error',
  123. source: 'postgres' as const,
  124. target: 'trigger:refresh_materialized_views',
  125. message: 'permission denied for relation user_documents',
  126. },
  127. {
  128. id: 'log-005',
  129. timestamp: '2024-06-20T13:45:00Z',
  130. level: 'info',
  131. source: 'auth' as const,
  132. target: 'email-confirmation',
  133. message: 'Sent verification email to alex@example.com',
  134. },
  135. ]
  136. function createMockedStudioTools() {
  137. const studioTools = getStudioTools()
  138. return Object.fromEntries(
  139. Object.entries(studioTools).map(([name, baseTool]) => {
  140. // Always mock execute_sql and deploy_edge_function with needsApproval disabled
  141. if (name === 'execute_sql') {
  142. return [name, { ...baseTool, needsApproval: false, execute: async () => [] as unknown[] }]
  143. }
  144. if (name === 'deploy_edge_function') {
  145. return [
  146. name,
  147. { ...baseTool, needsApproval: false, execute: async () => ({ success: true }) },
  148. ]
  149. }
  150. if (typeof baseTool.execute === 'function') {
  151. return [name, baseTool]
  152. }
  153. return [
  154. name,
  155. { ...baseTool, execute: async () => ({ status: 'Tool call mocked successfully.' }) },
  156. ]
  157. })
  158. ) as typeof studioTools
  159. }
  160. function createMockListTablesTool(overrideData?: Record<string, typeof MOCK_TABLES_DATA>) {
  161. return tool({
  162. description: 'Lists tables and columns for the provided schemas.',
  163. inputSchema: listTablesInputSchema,
  164. execute: async ({ schemas }: { schemas: string[] }) => {
  165. const effectiveSchemas = schemas?.length ? schemas : ['public']
  166. return effectiveSchemas.map((schema) => ({
  167. schema,
  168. tables: overrideData?.[schema] ?? MOCK_TABLES_DATA,
  169. }))
  170. },
  171. })
  172. }
  173. function createMockListExtensionsTool() {
  174. return tool({
  175. description: 'Lists installed database extensions.',
  176. inputSchema: z.object({}),
  177. execute: async () => {
  178. return MOCK_EXTENSIONS_DATA
  179. },
  180. })
  181. }
  182. function createMockListEdgeFunctionsTool() {
  183. return tool({
  184. description: 'Lists available Briven Edge Functions.',
  185. inputSchema: z.object({}),
  186. execute: async () => {
  187. return MOCK_EDGE_FUNCTIONS_DATA
  188. },
  189. })
  190. }
  191. function createMockGetAdvisorsTool() {
  192. return tool({
  193. description: 'Returns advisory notices for the project (mocked).',
  194. inputSchema: getAdvisorsInputSchema,
  195. execute: async ({ type }: { type?: 'security' | 'performance' }) => {
  196. if (type) {
  197. return MOCK_ADVISORIES_DATA.filter((advisory) => advisory.category === type)
  198. }
  199. return MOCK_ADVISORIES_DATA
  200. },
  201. })
  202. }
  203. function createMockGetLogsTool() {
  204. return tool({
  205. description: 'Fetches recent project logs for debugging or health checks (mocked).',
  206. inputSchema: getLogsInputSchema,
  207. execute: async ({
  208. limit = 10,
  209. level,
  210. source,
  211. search,
  212. }: {
  213. limit?: number
  214. level?: 'debug' | 'info' | 'warning' | 'error'
  215. source?: 'postgres' | 'auth' | 'storage' | 'edge_function'
  216. search?: string
  217. }) => {
  218. let filtered = MOCK_LOGS_DATA
  219. if (level) {
  220. filtered = filtered.filter((entry) => entry.level === level)
  221. }
  222. if (source) {
  223. filtered = filtered.filter((entry) => entry.source === source)
  224. }
  225. if (search) {
  226. const needle = search.toLowerCase()
  227. filtered = filtered.filter((entry) =>
  228. `${entry.message} ${entry.target}`.toLowerCase().includes(needle)
  229. )
  230. }
  231. return filtered.slice(0, limit)
  232. },
  233. })
  234. }
  235. function createMockListPoliciesTool() {
  236. return tool({
  237. description: 'Get existing RLS policies for provided schemas.',
  238. inputSchema: listPoliciesInputSchema,
  239. execute: async ({ schemas }: { schemas: string[] }) => {
  240. const effectiveSchemas = schemas?.length ? schemas : ['public']
  241. const results = [] as Array<{
  242. schema: string
  243. table: string
  244. policies: Array<{
  245. name: string
  246. command: 'select' | 'insert' | 'update' | 'delete'
  247. using?: string
  248. check?: string
  249. }>
  250. }>
  251. for (const schema of effectiveSchemas) {
  252. if (schema !== 'public') continue
  253. results.push(
  254. {
  255. schema,
  256. table: 'customers',
  257. policies: [
  258. {
  259. name: 'customers_tenant_select',
  260. command: 'select',
  261. using: "(auth.jwt() ->> 'tenant_id')::uuid = tenant_id",
  262. },
  263. ],
  264. },
  265. { schema, table: 'user_documents', policies: [] },
  266. { schema, table: 'projects', policies: [] }
  267. )
  268. }
  269. return results
  270. },
  271. })
  272. }
  273. export type MockToolOverrides = {
  274. list_tables?: Record<string, typeof MOCK_TABLES_DATA>
  275. }
  276. /**
  277. * Deterministic mock implementations of MCP/platform tools for evals.
  278. * These mirror tool names used in prompts so the model can call them,
  279. * but return stable, static data for repeatable tests.
  280. *
  281. * Note: search_docs uses the real implementation
  282. */
  283. export async function getMockTools(overrides?: MockToolOverrides) {
  284. const mockedStudioTools = createMockedStudioTools()
  285. const { search_docs } = await getMcpTools({
  286. accessToken: 'mock-access-token',
  287. projectRef: 'mock-project-ref',
  288. aiOptInLevel: 'schema_and_log_and_data',
  289. })
  290. assert(search_docs, 'search_docs tool not available from MCP server')
  291. return {
  292. ...mockedStudioTools,
  293. search_docs,
  294. list_tables: createMockListTablesTool(overrides?.list_tables),
  295. list_extensions: createMockListExtensionsTool(),
  296. list_edge_functions: createMockListEdgeFunctionsTool(),
  297. get_advisors: createMockGetAdvisorsTool(),
  298. get_logs: createMockGetLogsTool(),
  299. list_policies: createMockListPoliciesTool(),
  300. }
  301. }