diff --git a/apps/sim/connectors/google-drive/google-drive-errors.ts b/apps/sim/connectors/google-drive/google-drive-errors.ts index 28c2de91548..b57cd059b27 100644 --- a/apps/sim/connectors/google-drive/google-drive-errors.ts +++ b/apps/sim/connectors/google-drive/google-drive-errors.ts @@ -5,6 +5,10 @@ import { resolveRetryDelayMs, retryWithExponentialBackoff, } from '@/lib/knowledge/documents/utils' +import { + ConnectorSourceError, + type ConnectorSourceFailureCategory, +} from '@/connectors/source-error' import { readBodyWithLimit } from '@/connectors/utils' const GOOGLE_ERROR_BODY_MAX_BYTES = 64 * 1024 @@ -103,22 +107,47 @@ function classifyGoogleDriveError( return 'unknown' } -export class GoogleDriveApiError extends Error { +function diagnosticCategory( + kind: GoogleDriveErrorKind, + status: number +): ConnectorSourceFailureCategory | undefined { + switch (kind) { + case 'authorization': + case 'permission': + return 'authorization' + case 'not_found': + return 'source_unavailable' + case 'export_too_large': + case 'unsupported_export': + case 'policy': + return 'request_rejected' + case 'quota': + return 'rate_limit' + case 'transient': + return status === 429 || status === 403 ? 'rate_limit' : 'provider_unavailable' + default: + return undefined + } +} + +export class GoogleDriveApiError extends ConnectorSourceError { retryAfterMs?: number readonly reasons: readonly string[] readonly kind: GoogleDriveErrorKind readonly rateLimited: boolean - constructor( - readonly status: number, - normalizedReasons: readonly string[] - ) { + constructor(status: number, normalizedReasons: readonly string[]) { const diagnosticReasons = normalizedReasons.slice(0, GOOGLE_ERROR_REASON_MAX_COUNT) const reasonSuffix = diagnosticReasons.length > 0 ? ` (${diagnosticReasons.join(', ')})` : '' - super(`Google Drive API request failed with HTTP ${status}${reasonSuffix}`) + const kind = classifyGoogleDriveError(status, normalizedReasons) + super( + `Google Drive API request failed with HTTP ${status}${reasonSuffix}`, + status, + diagnosticCategory(kind, status) + ) this.name = 'GoogleDriveApiError' this.reasons = diagnosticReasons - this.kind = classifyGoogleDriveError(status, normalizedReasons) + this.kind = kind this.rateLimited = status === 429 || normalizedReasons.some((reason) => RATE_LIMIT_REASONS.has(reason)) } diff --git a/apps/sim/connectors/google-drive/google-drive.test.ts b/apps/sim/connectors/google-drive/google-drive.test.ts index b9f3e7fb71c..c093c2504b7 100644 --- a/apps/sim/connectors/google-drive/google-drive.test.ts +++ b/apps/sim/connectors/google-drive/google-drive.test.ts @@ -928,8 +928,12 @@ describe('Google Drive change feed', () => { { kind: 'removed', externalId: 'moved-out' }, { kind: 'removed', externalId: 'video' }, ]) - expect(result.nextCursor).toBe('5000') - expect(result.hasMore).toBe(false) + expect(result.nextCursor).toMatch(/^gdrive-shortcuts:v1:/) + expect(result.hasMore).toBe(true) + mockFetch.mockResolvedValueOnce(jsonResponse({ files: [] })) + await expect( + googleDriveConnector.listChanges!('token', {}, result.nextCursor!) + ).resolves.toEqual({ changes: [], nextCursor: '5000', hasMore: false }) const url = new URL(String(mockFetch.mock.calls[0][0])) expect(url.searchParams.get('pageToken')).toBe('4821') expect(url.searchParams.get('includeRemoved')).toBe('true') diff --git a/apps/sim/connectors/google-drive/google-drive.ts b/apps/sim/connectors/google-drive/google-drive.ts index 0fa102e9c21..cd551cd9066 100644 --- a/apps/sim/connectors/google-drive/google-drive.ts +++ b/apps/sim/connectors/google-drive/google-drive.ts @@ -7,6 +7,7 @@ import { driveFileAcl, type OpenSharingPolicy, } from '@/lib/knowledge/access/drive-permissions' +import type { MirroredDocumentAcl } from '@/lib/knowledge/access/types' import { OCR_IMAGE_MIME_TYPES } from '@/lib/knowledge/documents/ocr-request-policy' import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { drainGooglePagedList } from '@/lib/oauth/google-pagination' @@ -60,6 +61,13 @@ const GOOGLE_WORKSPACE_EXPORTS: Record = { 'application/vnd.google-apps.spreadsheet': XLSX_MIME_TYPE, 'application/vnd.google-apps.presentation': 'text/plain', } +const SHORTCUT_MIME_TYPE = 'application/vnd.google-apps.shortcut' +const SHORTCUT_FETCH_CONCURRENCY = 8 +const DRIVE_METADATA_MAX_BYTES = 1024 * 1024 +const DRIVE_PAGE_MAX_BYTES = 16 * 1024 * 1024 +const DRIVE_FILE_FIELDS = + 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,trashed,parents,shortcutDetails(targetId,targetMimeType,targetResourceKey)' + const FOLDER_MIME_TYPE = 'application/vnd.google-apps.folder' const SUPPORTED_TEXT_MIME_TYPES = [ @@ -100,6 +108,7 @@ function isSupportedTextFile(mimeType: string): boolean { } function rawFileType(file: DriveFile): { mimeType: string; fileName: string } | undefined { + if (file.mimeType.startsWith('application/vnd.google-apps.')) return undefined const byName = pipelineParsedMimeType(file.name) if (byName) return { mimeType: byName, fileName: file.name } if (OCR_IMAGE_MIME_TYPES.has(file.mimeType)) { @@ -124,7 +133,8 @@ function isSupportedFile(file: DriveFile): boolean { async function exportGoogleWorkspaceFile( accessToken: string, fileId: string, - sourceMimeType: string + sourceMimeType: string, + resourceKey?: string ): Promise { const exportMimeType = GOOGLE_WORKSPACE_EXPORTS[sourceMimeType] if (!exportMimeType) { @@ -137,7 +147,7 @@ async function exportGoogleWorkspaceFile( try { response = await fetchGoogleDriveWithRetry(url, { method: 'GET', - headers: { Authorization: `Bearer ${accessToken}` }, + headers: driveRequestHeaders(accessToken, fileId, resourceKey), }) } catch (error) { if (error instanceof GoogleDriveApiError && error.kind === 'export_too_large') { @@ -153,7 +163,11 @@ async function exportGoogleWorkspaceFile( return buffer } -async function downloadFile(accessToken: string, fileId: string): Promise { +async function downloadFile( + accessToken: string, + fileId: string, + resourceKey?: string +): Promise { // Listing runs with `includeItemsFromAllDrives`, so ids here can belong to a shared // drive; `supportsAllDrives` declares that support to `files.get` the same way the // metadata fetch in getDocument already does. (`files.export` takes no such param.) @@ -161,7 +175,7 @@ async function downloadFile(accessToken: string, fileId: string): Promise { +async function fetchFilePayload( + accessToken: string, + file: DriveFile, + resourceKey?: string +): Promise { if (GOOGLE_WORKSPACE_EXPORTS[file.mimeType]) { - const bytes = await exportGoogleWorkspaceFile(accessToken, file.id, file.mimeType) + const bytes = await exportGoogleWorkspaceFile(accessToken, file.id, file.mimeType, resourceKey) if (file.mimeType === 'application/vnd.google-apps.spreadsheet') { return { content: '', @@ -197,7 +215,7 @@ async function fetchFilePayload(accessToken: string, file: DriveFile): Promise '${lastSyncAt.toISOString()}'`) + if (lastSyncAt) { + /** Target edits do not modify the shortcut itself. */ + parts.push( + `(modifiedTime > '${lastSyncAt.toISOString()}' or mimeType = '${SHORTCUT_MIME_TYPE}')` + ) + } const fileType = (sourceConfig.fileType as string) || 'all' const mimeParts: string[] = [] @@ -460,6 +484,8 @@ function buildQuery( } } if (mimeParts.length > 0) { + /** Resolve the current target type: shortcutDetails.targetMimeType can be stale. */ + mimeParts.push(`mimeType = '${SHORTCUT_MIME_TYPE}'`) if (includeFolders) mimeParts.push(`mimeType = '${FOLDER_MIME_TYPE}'`) parts.push(`(${mimeParts.join(' or ')})`) } @@ -540,7 +566,8 @@ const DRIVE_PERMISSION_FIELDS = 'id,type,emailAddress,domain,role,allowFileDisco */ async function listFilePermissions( accessToken: string, - fileId: string + fileId: string, + resourceKey?: string ): Promise { const { items, truncated } = await drainGooglePagedList< DrivePermission, @@ -558,7 +585,7 @@ async function listFilePermissions( fetch: (url) => fetchGoogleDriveWithRetry(url, { method: 'GET', - headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + headers: driveRequestHeaders(accessToken, fileId, resourceKey), }), parseError: (response) => response.json().catch(() => null), getItems: (body) => body.permissions, @@ -583,15 +610,36 @@ async function listFilePermissions( async function resolveDriveAcls( accessToken: string, sourceConfig: Record, - externalIds: string[], + documents: readonly ExternalDocument[], syncContext?: Record -): Promise> { +): Promise> { const context = driveAclContext(sourceConfig, syncContext) if (!context) return {} - const acls: Record = {} - await mapWithConcurrency(externalIds, PERMISSION_FETCH_CONCURRENCY, async (fileId) => { + const acls: Record = {} + await mapWithConcurrency(documents, PERMISSION_FETCH_CONCURRENCY, async (document) => { + const fileId = document.externalId try { + if (document.metadata?.shortcutTargetId) { + const file = await readDriveFile(accessToken, fileId, undefined, true) + if (file.trashed) { + acls[fileId] = [] + return + } + if (file.mimeType === SHORTCUT_MIME_TYPE) { + const target = await readShortcutTarget(accessToken, file, true) + acls[fileId] = + target && isSupportedFile(target.file) + ? await shortcutAcl(accessToken, file, target.file, context, target.resourceKey) + : [] + return + } + const listedAcl = fileAcl(file, context) + if (listedAcl) { + acls[fileId] = listedAcl + return + } + } const permissions = await listFilePermissions(accessToken, fileId) acls[fileId] = driveFileAcl({ ...context, permissions }) } catch (error) { @@ -604,7 +652,11 @@ async function resolveDriveAcls( return acls } -function fileToStub(file: DriveFile, acl?: string[]): ExternalDocument { +function fileToStub( + file: DriveFile, + acl?: MirroredDocumentAcl, + target?: DriveFile +): ExternalDocument { /** * Sheets moved from a first-sheet-only CSV export to the complete XLSX source. * The namespace forces one rehydration for existing rows whose old hash would @@ -621,18 +673,260 @@ function fileToStub(file: DriveFile, acl?: string[]): ExternalDocument { ...(acl ? { acl } : {}), mimeType: 'text/plain', sourceUrl: file.webViewLink || `https://drive.google.com/file/d/${file.id}/view`, - contentHash: `${hashNamespace}:${file.id}:${file.modifiedTime ?? ''}`, + contentHash: target + ? `gdrive:shortcut:v1:${file.id}:${file.modifiedTime ?? ''}:${target.id}:${target.modifiedTime ?? ''}:${target.mimeType}` + : `${hashNamespace}:${file.id}:${file.modifiedTime ?? ''}`, metadata: { - originalMimeType: file.mimeType, - modifiedTime: file.modifiedTime, + originalMimeType: target?.mimeType ?? file.mimeType, + ...(target ? { shortcutTargetId: target.id } : {}), + modifiedTime: target?.modifiedTime ?? file.modifiedTime, createdTime: file.createdTime, owners: file.owners?.map((o) => o.displayName || o.emailAddress).filter(Boolean), starred: file.starred, - fileSize: file.size ? Number(file.size) : undefined, + fileSize: (target ?? file).size ? Number((target ?? file).size) : undefined, }, } } +/** Resource keys are capabilities: send only to Drive, never persist them in document metadata. */ +function driveRequestHeaders( + accessToken: string, + fileId: string, + resourceKey?: string +): Record { + if ( + resourceKey !== undefined && + (!/^[A-Za-z0-9_-]{1,1024}$/.test(resourceKey) || !/^[A-Za-z0-9_-]{1,1024}$/.test(fileId)) + ) { + throw new Error('Google Drive returned malformed resource-key metadata') + } + return { + Authorization: `Bearer ${accessToken}`, + ...(resourceKey ? { 'X-Goog-Drive-Resource-Keys': `${fileId}/${resourceKey}` } : {}), + } +} + +async function readDriveJson(response: Response, limitBytes: number): Promise { + const body = await readBodyWithLimit(response, limitBytes) + if (!body) throw new Error('Google Drive metadata exceeded its size limit') + try { + return JSON.parse(body.toString('utf8')) + } catch { + throw new Error('Google Drive returned malformed metadata') + } +} + +async function readDriveFile( + accessToken: string, + fileId: string, + resourceKey?: string, + permissions = false +): Promise { + const fields = `${DRIVE_FILE_FIELDS}${permissions ? `,permissions(${DRIVE_PERMISSION_FIELDS})` : ''}` + const response = await fetchGoogleDriveWithRetry( + `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(fileId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`, + { method: 'GET', headers: driveRequestHeaders(accessToken, fileId, resourceKey) } + ) + return parseDriveFileMetadata(await readDriveJson(response, DRIVE_METADATA_MAX_BYTES), fileId) +} + +/** Resolve one current file target; folder traversal and shortcut chains are not expanded. */ +async function readShortcutTarget( + accessToken: string, + shortcut: DriveFile, + permissions = false +): Promise<{ file: DriveFile; resourceKey?: string } | null> { + const details = shortcut.shortcutDetails + if ( + !details || + typeof details.targetId !== 'string' || + !/^[A-Za-z0-9_-]{1,1024}$/.test(details.targetId) || + (details.targetResourceKey !== undefined && typeof details.targetResourceKey !== 'string') + ) { + throw new Error('Google Drive returned malformed shortcut metadata') + } + if (details.targetId === shortcut.id) throw new Error('Google Drive returned a cyclic shortcut') + try { + const file = await readDriveFile( + accessToken, + details.targetId, + details.targetResourceKey, + permissions + ) + if (!file.trashed && !isDriveFileListItem(file)) { + throw new Error('Google Drive returned malformed shortcut target metadata') + } + return file.trashed ? null : { file, resourceKey: details.targetResourceKey } + } catch (error) { + if ( + error instanceof GoogleDriveApiError && + (error.kind === 'not_found' || error.kind === 'permission') + ) + return null + throw error + } +} + +function unavailableShortcut(file: DriveFile): ExternalDocument { + return { + ...markSkipped( + fileToStub(file, []), + 'Shortcut target is unavailable or the connector account cannot access it' + ), + skippedExistingDisposition: 'replace', + } +} + +/** A shortcut's own grants never grant access to its target's content. */ +async function shortcutAcl( + accessToken: string, + shortcut: DriveFile, + target: DriveFile, + context: DriveAclContext, + resourceKey?: string +): Promise { + return { + acl: + fileAcl(shortcut, context) ?? + driveFileAcl({ + ...context, + permissions: await listFilePermissions(accessToken, shortcut.id), + }), + requirements: [ + fileAcl(target, context) ?? + driveFileAcl({ + ...context, + permissions: await listFilePermissions(accessToken, target.id, resourceKey), + }), + ], + } +} + +async function listedFileToDocument( + accessToken: string, + sourceConfig: Record, + file: DriveFile, + syncContext?: Record +): Promise { + if (file.trashed || file.mimeType === FOLDER_MIME_TYPE) return null + const context = driveAclContext(sourceConfig, syncContext) + let target: DriveFile | undefined + let acl: MirroredDocumentAcl | undefined = fileAcl(file, context) + if (file.mimeType === SHORTCUT_MIME_TYPE) { + let resolved: Awaited> + try { + resolved = await readShortcutTarget(accessToken, file, Boolean(context)) + } catch (error) { + /** Member visibility requires verified target access; a failed page never grants it. */ + if (isPerMemberListing(syncContext)) throw error + logger.warn( + 'Could not resolve shortcut target; deferring to content hydration', + googleDriveErrorLogFields(error) + ) + return fileToStub(file, []) + } + if (!resolved) return isPerMemberListing(syncContext) ? null : unavailableShortcut(file) + target = resolved.file + if (context) { + try { + acl = await shortcutAcl(accessToken, file, target, context, resolved.resourceKey) + } catch (error) { + logger.warn( + 'Could not verify shortcut and target permissions', + googleDriveErrorLogFields(error) + ) + acl = [] + } + } + } + const contentFile = target ?? file + if (!matchesFileType((sourceConfig.fileType as string) || 'all', contentFile)) return null + return stubOrSkipBySize( + fileToStub(file, acl, target), + Number(contentFile.size) || undefined, + CONNECTOR_MAX_FILE_BYTES + ) +} + +const SHORTCUT_CHANGE_CURSOR_PREFIX = 'gdrive-shortcuts:v1:' +interface ShortcutChangeCursor { + resume: string + pageToken?: string +} + +function writeShortcutChangeCursor(cursor: ShortcutChangeCursor): string { + return `${SHORTCUT_CHANGE_CURSOR_PREFIX}${Buffer.from(JSON.stringify(cursor)).toString('base64url')}` +} + +function readShortcutChangeCursor(cursor: string): ShortcutChangeCursor | undefined { + if (!cursor.startsWith(SHORTCUT_CHANGE_CURSOR_PREFIX)) return undefined + try { + if (cursor.length > 65536) throw new Error() + const value: unknown = JSON.parse( + Buffer.from(cursor.slice(SHORTCUT_CHANGE_CURSOR_PREFIX.length), 'base64url').toString('utf8') + ) + if ( + !isPlainRecord(value) || + typeof value.resume !== 'string' || + !value.resume || + value.resume.length > 16384 || + (value.pageToken !== undefined && + (typeof value.pageToken !== 'string' || !value.pageToken || value.pageToken.length > 16384)) + ) + throw new Error() + return { resume: value.resume, pageToken: value.pageToken } + } catch { + throw new InvalidDriveListingCursor('Google Drive shortcut listing must restart') + } +} + +/** Target edits and revocations need not emit a change for their shortcuts. */ +async function listShortcutChanges( + accessToken: string, + sourceConfig: Record, + cursor: ShortcutChangeCursor +): Promise { + const params = new URLSearchParams({ + q: `trashed = false and mimeType = '${SHORTCUT_MIME_TYPE}'`, + orderBy: 'createdTime', + fields: `kind,nextPageToken,incompleteSearch,files(${DRIVE_FILE_FIELDS})`, + pageSize: '100', + supportsAllDrives: 'true', + includeItemsFromAllDrives: 'true', + }) + if (cursor.pageToken) params.set('pageToken', cursor.pageToken) + const response = await fetchGoogleDriveWithRetry( + `https://www.googleapis.com/drive/v3/files?${params}`, + { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + } + ) + const page = parseDriveFileListResponse(await readDriveJson(response, DRIVE_PAGE_MAX_BYTES)) + if (page.incompleteSearch) throw new Error('Google Drive shortcut search was incomplete') + if (page.nextPageToken && page.nextPageToken === cursor.pageToken) + throw new Error('Google Drive repeated a shortcut continuation token') + const changes = await mapWithConcurrency( + page.files, + SHORTCUT_FETCH_CONCURRENCY, + async (file): Promise => { + const document = await listedFileToDocument(accessToken, sourceConfig, file, { + perMemberListing: true, + }) + return document + ? { kind: 'upsert', externalId: file.id, document } + : { kind: 'removed', externalId: file.id } + } + ) + return { + changes, + nextCursor: page.nextPageToken + ? writeShortcutChangeCursor({ ...cursor, pageToken: page.nextPageToken }) + : cursor.resume, + hasMore: Boolean(page.nextPageToken), + } +} + const TREE_CURSOR_PREFIX = 'gdrive-tree:1:' const MAX_TREE_CURSOR_BYTES = 512 * 1024 const MAX_PENDING_FOLDERS = 10_000 @@ -754,7 +1048,7 @@ export const googleDriveConnector: ConnectorConfig = { * Permissions ride along only where the run mirrors them. Every other * crawl would pull a permission array per file and discard it. */ - fields: `kind,nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,parents${ + fields: `kind,nextPageToken,incompleteSearch,files(${DRIVE_FILE_FIELDS}${ aclContext ? `,permissions(${DRIVE_PERMISSION_FIELDS})` : '' })`, supportsAllDrives: 'true', @@ -796,7 +1090,7 @@ export const googleDriveConnector: ConnectorConfig = { throw error } - const data = parseDriveFileListResponse(await response.json()) + const data = parseDriveFileListResponse(await readDriveJson(response, DRIVE_PAGE_MAX_BYTES)) const files = data.files /** @@ -829,19 +1123,10 @@ export const googleDriveConnector: ConnectorConfig = { } } - const pageDocuments = files - .filter( - (f) => - f.mimeType !== FOLDER_MIME_TYPE && - matchesFileType((sourceConfig.fileType as string) || 'all', f) - ) - .map((f) => - stubOrSkipBySize( - fileToStub(f, fileAcl(f, aclContext)), - Number(f.size) || undefined, - CONNECTOR_MAX_FILE_BYTES - ) - ) + const resolved = await mapWithConcurrency(files, SHORTCUT_FETCH_CONCURRENCY, (file) => + listedFileToDocument(accessToken, sourceConfig, file, syncContext) + ) + const pageDocuments = resolved.filter((doc): doc is ExternalDocument => doc !== null) const page = takeIndexableWithinCap( pageDocuments, @@ -891,74 +1176,60 @@ export const googleDriveConnector: ConnectorConfig = { ), getDocumentAcls: (accessToken, sourceConfig, documents, syncContext) => - resolveDriveAcls( - accessToken, - sourceConfig, - documents.map((doc) => doc.externalId), - syncContext - ), + resolveDriveAcls(accessToken, sourceConfig, documents, syncContext), getDocument: async ( accessToken: string, sourceConfig: Record, externalId: string ): Promise => { - const fields = - 'id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,trashed' - const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(externalId)}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true` - - let response: Response + let file: DriveFile try { - response = await fetchGoogleDriveWithRetry(url, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - }, - }) + file = await readDriveFile(accessToken, externalId) } catch (error) { - if (!(error instanceof GoogleDriveApiError)) throw error - if (error.kind === 'not_found') return null + if (error instanceof GoogleDriveApiError && error.kind === 'not_found') return null throw error } - - const file = parseDriveFileMetadata(await response.json(), externalId) - if (file.trashed) return null + let contentFile = file + let resourceKey: string | undefined + if (file.mimeType === SHORTCUT_MIME_TYPE) { + const resolved = await readShortcutTarget(accessToken, file) + if (!resolved) return unavailableShortcut(file) + contentFile = resolved.file + resourceKey = resolved.resourceKey + } + const stub = fileToStub(file, undefined, contentFile === file ? undefined : contentFile) /** * Mirrors the listing filter. The marker distinguishes a successfully * verified unindexable file from an ambiguous null hydration. */ - if (!isSupportedFile(file)) { + if (!matchesFileType((sourceConfig.fileType as string) || 'all', contentFile)) { logger.info('Google Drive file has no extractable text type', { fileId: file.id, mimeType: file.mimeType, }) return { - ...markSkipped(fileToStub(file), 'File is no longer an indexable document'), + ...markSkipped(stub, 'File is no longer an indexable document'), skippedExistingDisposition: 'replace', } } try { - const payload = await fetchFilePayload(accessToken, file) + const payload = await fetchFilePayload(accessToken, contentFile, resourceKey) if (!payload.content.trim() && !payload.sourceFile?.bytes.length) { return { - ...markSkipped( - { ...fileToStub(file), ...payload }, - 'Document contains no extractable text' - ), + ...markSkipped({ ...stub, ...payload }, 'Document contains no extractable text'), skippedExistingDisposition: 'replace', } } - const stub = fileToStub(file) return { ...stub, ...payload, contentDeferred: false } } catch (error) { if (error instanceof ConnectorFileTooLargeError) { logger.info('Skipping oversized Google Drive file', { fileId: file.id, name: file.name }) - return markSkipped(fileToStub(file), sizeLimitSkipReason(error.limitBytes)) + return markSkipped(stub, sizeLimitSkipReason(error.limitBytes)) } /** * The file exists but its content could not be read. Propagate so the engine @@ -1119,6 +1390,10 @@ export const googleDriveConnector: ConnectorConfig = { sourceConfig: Record, cursor: string ): Promise => { + const shortcutCursor = readShortcutChangeCursor(cursor) + if (shortcutCursor) { + return listShortcutChanges(accessToken, sourceConfig, shortcutCursor) + } const queryParams = new URLSearchParams({ pageToken: cursor, pageSize: '100', @@ -1127,8 +1402,7 @@ export const googleDriveConnector: ConnectorConfig = { includeItemsFromAllDrives: 'true', restrictToMyDrive: 'false', spaces: 'drive', - fields: - 'nextPageToken,newStartPageToken,changes(changeType,removed,fileId,file(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,trashed,parents))', + fields: `nextPageToken,newStartPageToken,changes(changeType,removed,fileId,file(${DRIVE_FILE_FIELDS}))`, }) const url = `https://www.googleapis.com/drive/v3/changes?${queryParams.toString()}` @@ -1143,20 +1417,36 @@ export const googleDriveConnector: ConnectorConfig = { throw error } - const data = parseDriveChangeListResponse(await response.json()) - const changes: ExternalChange[] = [] - for (const change of data.changes) { - const mapped = driveChangeToExternal(change, sourceConfig) - if (mapped) changes.push(mapped) - } + const data = parseDriveChangeListResponse(await readDriveJson(response, DRIVE_PAGE_MAX_BYTES)) + const mappedChanges = await mapWithConcurrency( + data.changes, + SHORTCUT_FETCH_CONCURRENCY, + async (change) => { + if (!change.removed && change.file?.mimeType === SHORTCUT_MIME_TYPE) { + const document = await listedFileToDocument(accessToken, sourceConfig, change.file, { + perMemberListing: true, + }) + return document + ? { kind: 'upsert' as const, externalId: change.fileId!, document } + : { kind: 'removed' as const, externalId: change.fileId! } + } + return driveChangeToExternal(change, sourceConfig) + } + ) + const changes = mappedChanges.filter((change): change is ExternalChange => change !== null) const nextCursor = data.nextPageToken ?? data.newStartPageToken if (!nextCursor) { throw new Error('Google Drive API returned malformed change-list metadata') } - return { changes, nextCursor, hasMore: Boolean(data.nextPageToken) } + return { + changes, + nextCursor: data.nextPageToken ?? writeShortcutChangeCursor({ resume: nextCursor }), + hasMore: true, + } }, - isChangeCursorInvalidError: isDriveChangeCursorInvalidError, + isChangeCursorInvalidError: (error) => + error instanceof InvalidDriveListingCursor || isDriveChangeCursorInvalidError(error), isListingCursorInvalidError: (error) => error instanceof InvalidDriveListingCursor || isDriveChangeCursorInvalidError(error), } diff --git a/apps/sim/connectors/google-drive/shortcuts.test.ts b/apps/sim/connectors/google-drive/shortcuts.test.ts new file mode 100644 index 00000000000..b1c110184b6 --- /dev/null +++ b/apps/sim/connectors/google-drive/shortcuts.test.ts @@ -0,0 +1,352 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { fetchMock } = vi.hoisted(() => ({ fetchMock: vi.fn() })) +vi.mock('@/components/icons', () => ({ GoogleDriveIcon: () => null })) + +import { googleDriveConnector as drive } from '@/connectors/google-drive/google-drive' +import { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' +import { CONNECTOR_MAX_FILE_BYTES } from '@/connectors/utils' + +const shortcutMime = 'application/vnd.google-apps.shortcut' +const docMime = 'application/vnd.google-apps.document' +const json = (body: unknown) => + new Response(JSON.stringify(body), { headers: { 'Content-Type': 'application/json' } }) +const denied = (reason = 'notFound', status = 404) => + new Response(JSON.stringify({ error: { errors: [{ reason }] } }), { status }) +function file(overrides: Record = {}) { + return { + id: 'target', + name: 'Target.pdf', + mimeType: 'application/pdf', + modifiedTime: '2026-01-01T00:00:00Z', + ...overrides, + } +} +function shortcut(overrides: Record = {}) { + return file({ + id: 'shortcut', + name: 'Alias.pdf', + mimeType: shortcutMime, + shortcutDetails: { targetId: 'target', targetMimeType: 'application/pdf' }, + ...overrides, + }) +} +const urlAt = (index: number) => new URL(String(fetchMock.mock.calls[index][0])) + +describe('Drive file shortcuts', () => { + beforeEach(() => { + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('downloads the target PDF and keeps the shortcut identity and listing hash', async () => { + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()] })) + .mockResolvedValueOnce(json(file())) + const listing = await drive.listDocuments('token', {}) + expect(listing.documents).toHaveLength(1) + expect(fetchMock).toHaveBeenCalledTimes(2) + fetchMock + .mockResolvedValueOnce(json(shortcut())) + .mockResolvedValueOnce(json(file())) + .mockResolvedValueOnce(new Response('%PDF-fixture')) + const hydrated = await drive.getDocument('token', {}, 'shortcut') + expect(hydrated).toMatchObject({ + externalId: 'shortcut', + title: 'Alias.pdf', + contentHash: listing.documents[0].contentHash, + contentDeferred: false, + mimeType: 'application/pdf', + sourceFile: { fileName: 'Target.pdf', bytes: Buffer.from('%PDF-fixture') }, + }) + expect(urlAt(4).pathname).toBe('/drive/v3/files/target') + expect(urlAt(4).searchParams.get('alt')).toBe('media') + }) + + it('uses current target MIME and filename instead of stale shortcut hints', async () => { + fetchMock + .mockResolvedValueOnce(json(shortcut())) + .mockResolvedValueOnce(json(file({ name: 'Native document', mimeType: docMime }))) + .mockResolvedValueOnce(new Response('Native content')) + expect(await drive.getDocument('token', {}, 'shortcut')).toMatchObject({ + content: 'Native content', + mimeType: 'text/plain', + }) + expect(urlAt(2).pathname).toBe('/drive/v3/files/target/export') + }) + + it('sends resource keys to target metadata and content only, without persisting them', async () => { + const alias = shortcut({ + shortcutDetails: { targetId: 'target', targetResourceKey: 'synthetic-resource-key' }, + }) + fetchMock + .mockResolvedValueOnce(json(alias)) + .mockResolvedValueOnce(json(file())) + .mockResolvedValueOnce(new Response('pdf')) + const hydrated = await drive.getDocument('token', {}, 'shortcut') + expect(new Headers(fetchMock.mock.calls[0][1].headers).has('X-Goog-Drive-Resource-Keys')).toBe( + false + ) + for (const index of [1, 2]) + expect( + new Headers(fetchMock.mock.calls[index][1].headers).get('X-Goog-Drive-Resource-Keys') + ).toBe('target/synthetic-resource-key') + expect(JSON.stringify(hydrated)).not.toContain('synthetic-resource-key') + }) + + it('observes target edits during incremental listings without downloading unchanged content', async () => { + const hashes = [] + for (const modifiedTime of ['2026-01-01', '2026-01-02']) { + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()] })) + .mockResolvedValueOnce(json(file({ modifiedTime }))) + const page = await drive.listDocuments('token', {}, undefined, {}, new Date('2026-01-01')) + hashes.push(page.documents[0].contentHash) + } + expect(hashes[0]).not.toBe(hashes[1]) + expect(urlAt(0).searchParams.get('q')).toContain(`or mimeType = '${shortcutMime}'`) + expect( + fetchMock.mock.calls.every(([url]) => !new URL(String(url)).searchParams.has('alt')) + ).toBe(true) + }) + + it('includes shortcuts in type-filtered queries and filters on the current target', async () => { + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()] })) + .mockResolvedValueOnce(json(file({ mimeType: docMime }))) + const page = await drive.listDocuments('token', { fileType: 'documents' }) + expect(page.documents).toHaveLength(1) + expect(urlAt(0).searchParams.get('q')).toContain(`mimeType = '${shortcutMime}'`) + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()] })) + .mockResolvedValueOnce(json(file())) + expect((await drive.listDocuments('token', { fileType: 'documents' })).documents).toEqual([]) + }) + + it.each(['application/vnd.google-apps.folder', shortcutMime, 'application/vnd.google-apps.form'])( + 'does not download unsupported native %s with a PDF filename', + async (mimeType) => { + fetchMock + .mockResolvedValueOnce(json(shortcut())) + .mockResolvedValueOnce(json(file({ mimeType }))) + expect(await drive.getDocument('token', {}, 'shortcut')).toMatchObject({ + skippedExistingDisposition: 'replace', + skippedReason: 'File is no longer an indexable document', + }) + expect(fetchMock).toHaveBeenCalledTimes(2) + } + ) + + it.each([ + { targetId: 'shortcut' }, + { targetId: 'other', targetResourceKey: 'bad\r\nheader' }, + { targetId: '' }, + null, + ])( + 'rejects malformed or cyclic target metadata before requesting it', + async (shortcutDetails) => { + fetchMock.mockResolvedValueOnce(json(shortcut({ shortcutDetails }))) + await expect(drive.getDocument('token', {}, 'shortcut')).rejects.toThrow(/malformed|cyclic/) + expect(fetchMock).toHaveBeenCalledTimes(1) + } + ) + + it.each([ + ['notFound', 404], + ['insufficientFilePermissions', 403], + ] as const)( + 'withdraws member visibility and clears unavailable target content for %s', + async (reason, status) => { + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()] })) + .mockResolvedValueOnce(denied(reason, status)) + expect( + (await drive.listDocuments('token', {}, undefined, { perMemberListing: true })).documents + ).toEqual([]) + fetchMock + .mockResolvedValueOnce(json(shortcut())) + .mockResolvedValueOnce(denied(reason, status)) + expect(await drive.getDocument('token', {}, 'shortcut')).toMatchObject({ + skippedExistingDisposition: 'replace', + acl: [], + skippedReason: expect.stringContaining('Shortcut target is unavailable'), + }) + } + ) + + it('keeps other source documents progressing when target metadata fails', async () => { + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut(), file({ id: 'ordinary' })] })) + .mockResolvedValueOnce(denied('invalid', 400)) + const page = await drive.listDocuments('token', {}) + expect(page.documents.map((doc) => doc.externalId)).toEqual(['shortcut', 'ordinary']) + expect(page.documents[0]).toMatchObject({ contentDeferred: true, acl: [] }) + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()] })) + .mockResolvedValueOnce(denied('invalid', 400)) + await expect( + drive.listDocuments('token', {}, undefined, { perMemberListing: true }) + ).rejects.toBeInstanceOf(GoogleDriveApiError) + }) + + it('requires both shortcut and target permissions when mirroring ACLs', async () => { + const alias = shortcut({ + permissions: [{ type: 'user', emailAddress: 'alice@fixture.test', role: 'reader' }], + }) + const target = file({ + permissions: [{ type: 'group', emailAddress: 'team@fixture.test', role: 'reader' }], + }) + fetchMock.mockResolvedValueOnce(json({ files: [alias] })).mockResolvedValueOnce(json(target)) + const page = await drive.listDocuments( + 'token', + { adminEmail: 'admin@fixture.test' }, + undefined, + { mirrorsSourceAcls: true } + ) + expect(page.documents[0].acl).toEqual({ + acl: ['u:alice@fixture.test'], + requirements: [['g:google-drive:fixture.test:team@fixture.test']], + }) + }) + + it('preserves the target restriction through the separate ACL lookup hook', async () => { + const alias = shortcut({ + permissions: [{ type: 'user', emailAddress: 'alice@fixture.test', role: 'reader' }], + }) + const target = file({ + permissions: [{ type: 'user', emailAddress: 'bob@fixture.test', role: 'reader' }], + }) + fetchMock.mockResolvedValueOnce(json({ files: [alias] })).mockResolvedValueOnce(json(target)) + const page = await drive.listDocuments('token', {}) + expect(page.documents[0].acl).toBeUndefined() + fetchMock.mockResolvedValueOnce(json(alias)).mockResolvedValueOnce(json(target)) + const acls = await drive.getDocumentAcls!( + 'token', + { adminEmail: 'admin@fixture.test' }, + page.documents, + { mirrorsSourceAcls: true } + ) + expect(acls.shortcut).toEqual({ + acl: ['u:alice@fixture.test'], + requirements: [['u:bob@fixture.test']], + }) + fetchMock.mockResolvedValueOnce(json(alias)).mockResolvedValueOnce(denied()) + expect( + await drive.getDocumentAcls!('token', { adminEmail: 'admin@fixture.test' }, page.documents, { + mirrorsSourceAcls: true, + }) + ).toEqual({ shortcut: [] }) + }) + + it('fetches target permissions with its resource key and fails closed on permission lookup errors', async () => { + fetchMock + .mockResolvedValueOnce( + json({ + files: [ + shortcut({ + permissions: [{ type: 'anyone', role: 'reader', allowFileDiscovery: true }], + shortcutDetails: { targetId: 'target', targetResourceKey: 'key' }, + }), + ], + }) + ) + .mockResolvedValueOnce(json(file())) + .mockResolvedValueOnce(denied('insufficientFilePermissions', 403)) + const page = await drive.listDocuments( + 'token', + { adminEmail: 'admin@fixture.test', openSharing: 'anyone' }, + undefined, + { mirrorsSourceAcls: true } + ) + expect(page.documents[0].acl).toEqual([]) + expect(urlAt(2).pathname).toBe('/drive/v3/files/target/permissions') + expect(new Headers(fetchMock.mock.calls[2][1].headers).get('X-Goog-Drive-Resource-Keys')).toBe( + 'target/key' + ) + }) + + it('checks target size at listing and caps target bytes while downloading', async () => { + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()] })) + .mockResolvedValueOnce(json(file({ size: String(CONNECTOR_MAX_FILE_BYTES + 1) }))) + expect((await drive.listDocuments('token', {})).documents[0].skippedReason).toContain('exceeds') + fetchMock + .mockResolvedValueOnce(json(shortcut())) + .mockResolvedValueOnce(json(file())) + .mockResolvedValueOnce( + new Response('small', { + headers: { 'Content-Length': String(CONNECTOR_MAX_FILE_BYTES + 1) }, + }) + ) + expect((await drive.getDocument('token', {}, 'shortcut'))?.skippedReason).toContain('exceeds') + }) + + it('caps metadata bytes before parsing', async () => { + fetchMock + .mockResolvedValueOnce(json(shortcut())) + .mockResolvedValueOnce( + new Response('{}', { headers: { 'Content-Length': String(1024 * 1024 + 1) } }) + ) + await expect(drive.getDocument('token', {}, 'shortcut')).rejects.toThrow('metadata exceeded') + }) + + it('bounds shortcut metadata concurrency to eight without fetching ordinary files', async () => { + let active = 0 + let peak = 0 + fetchMock.mockImplementation(async (input: string) => { + const url = new URL(input) + if (url.pathname.endsWith('/files')) + return json({ + files: Array.from({ length: 30 }, (_, index) => shortcut({ id: `alias-${index}` })), + }) + active++ + peak = Math.max(peak, active) + await Promise.resolve() + active-- + return json(file()) + }) + expect((await drive.listDocuments('token', {})).documents).toHaveLength(30) + expect(peak).toBeLessThanOrEqual(8) + }) + + it('durably sweeps shortcuts after an empty change feed to find target updates and revocations', async () => { + fetchMock.mockResolvedValueOnce(json({ changes: [], newStartPageToken: 'resume' })) + const changes = await drive.listChanges!('token', {}, 'start') + expect(changes.hasMore).toBe(true) + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut()], nextPageToken: 'page-two' })) + .mockResolvedValueOnce(json(file())) + const first = await drive.listChanges!('token', {}, changes.nextCursor!) + expect(first).toMatchObject({ + hasMore: true, + changes: [{ kind: 'upsert', externalId: 'shortcut' }], + }) + expect(urlAt(1).searchParams.get('q')).toBe(`trashed = false and mimeType = '${shortcutMime}'`) + fetchMock + .mockResolvedValueOnce(json({ files: [shortcut({ id: 'revoked' })] })) + .mockResolvedValueOnce(denied()) + const second = await drive.listChanges!('token', {}, first.nextCursor!) + expect(second).toEqual({ + changes: [{ kind: 'removed', externalId: 'revoked' }], + hasMore: false, + nextCursor: 'resume', + }) + expect(urlAt(3).searchParams.get('pageToken')).toBe('page-two') + }) + + it('rejects corrupt shortcut cursors and incomplete searches without advancing the feed', async () => { + await expect(drive.listChanges!('token', {}, 'gdrive-shortcuts:v1:invalid')).rejects.toThrow( + 'must restart' + ) + expect(fetchMock).not.toHaveBeenCalled() + fetchMock.mockResolvedValueOnce(json({ changes: [], newStartPageToken: 'resume' })) + const page = await drive.listChanges!('token', {}, 'start') + fetchMock.mockResolvedValueOnce(json({ files: [], incompleteSearch: true })) + await expect(drive.listChanges!('token', {}, page.nextCursor!)).rejects.toThrow('incomplete') + }) +}) diff --git a/apps/sim/connectors/source-error.ts b/apps/sim/connectors/source-error.ts new file mode 100644 index 00000000000..dd4745021d5 --- /dev/null +++ b/apps/sim/connectors/source-error.ts @@ -0,0 +1,19 @@ +/** Safe provider-owned classification, independent of HTTP status or free-form messages. */ +export type ConnectorSourceFailureCategory = + | 'authorization' + | 'source_unavailable' + | 'request_rejected' + | 'rate_limit' + | 'provider_unavailable' + +/** Providers classify their structured reasons here; shared diagnostics own user-facing text. */ +export class ConnectorSourceError extends Error { + constructor( + message: string, + readonly status: number, + readonly category?: ConnectorSourceFailureCategory + ) { + super(message) + this.name = 'ConnectorSourceError' + } +} diff --git a/apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts b/apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts new file mode 100644 index 00000000000..11a8836ba91 --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts @@ -0,0 +1,359 @@ +/** Real sync workers, PostgreSQL, file storage, PDF parsing, indexing, member observations and application authorization; Drive and embedding responses are synthetic. */ +import { mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeConnector, + knowledgeConnectorMember, + organization, + user, + workspace, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' +import { PDFDocument, StandardFonts } from 'pdf-lib' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +const fixture = vi.hoisted(() => ({ storageRoot: '', embeddingCalls: 0 })) +vi.mock('@/lib/uploads/core/setup.server', () => ({ + get UPLOAD_DIR_SERVER() { + return fixture.storageRoot + }, +})) +vi.mock('@/lib/embeddings', async () => ({ + ...(await import('@/lib/embeddings/client')), + assertKnowledgeEmbeddingCapacity: async () => {}, + embedKnowledge: async (texts: string[]) => { + fixture.embeddingCalls++ + return { + embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), + totalTokens: texts.length, + billableTokens: 0, + isBYOK: true, + modelName: 'text-embedding-3-small', + pricingId: 'text-embedding-3-small', + } + }, +})) + +import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, + seedKnowledgeMemberFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' +import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import * as connectorTokens from '@/lib/knowledge/connectors/access-token' +import * as memberAccess from '@/lib/knowledge/connectors/member-access' +import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' +import { executeSync } from '@/lib/knowledge/connectors/sync-engine' +import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' + +const json = (body: unknown) => Response.json(body) +const denied = () => Response.json({ error: { errors: [{ reason: 'notFound' }] } }, { status: 404 }) + +describe('Drive shortcuts through indexing and search', () => { + const ids = createKnowledgeAclFixtureIds() + let billing: Awaited> + let pdfBytes: Buffer + let revision = 1 + let targetMissing = false + let denyBob = false + const aliasPresent = true + let permittedTargetUser: string + let targetDownloads = 0 + let aliasDownloads = 0 + let enrolled: Awaited> + const principal = (userId: string) => ({ + kind: 'session' as const, + userId, + sessionId: 'shortcut-fixture', + }) + const permission = (userId: string) => ({ + type: 'user', + emailAddress: `${userId}@fixture.test`, + role: 'reader', + }) + const alias = () => ({ + id: 'shortcut', + name: 'Alias.pdf', + mimeType: 'application/vnd.google-apps.shortcut', + modifiedTime: '2026-01-01T00:00:00Z', + parents: ['root'], + shortcutDetails: { + targetId: 'target', + targetMimeType: 'application/pdf', + targetResourceKey: 'fixture-key', + }, + permissions: [permission(ids.aliceId), permission(ids.bobId)], + }) + const target = () => ({ + id: 'target', + name: 'Current.pdf', + mimeType: 'application/pdf', + modifiedTime: `2026-01-0${revision}T00:00:00Z`, + size: String(pdfBytes.length), + permissions: [permission(permittedTargetUser)], + }) + + async function setPdf(text: string) { + const pdf = await PDFDocument.create() + const font = await pdf.embedFont(StandardFonts.Helvetica) + pdf.addPage().drawText(text, { x: 20, y: 500, size: 12, font }) + pdfBytes = Buffer.from(await pdf.save()) + } + async function providerFetch(input: string | URL | Request, init?: RequestInit) { + const url = new URL( + typeof input === 'string' ? input : input instanceof URL ? input.href : input.url + ) + const headers = new Headers(init?.headers) + if (url.hostname === 'admin.googleapis.com') { + if (url.pathname.endsWith('/groups')) return json({ groups: [] }) + if (url.pathname.endsWith('/domains')) return json({ domains: [] }) + } + if (url.hostname !== 'www.googleapis.com') throw new Error('Unexpected fixture provider') + if (url.pathname.endsWith('/changes/startPageToken')) return json({ startPageToken: 'start' }) + if (url.pathname.endsWith('/changes')) return json({ changes: [], newStartPageToken: 'resume' }) + if (url.pathname.endsWith('/files')) return json({ files: aliasPresent ? [alias()] : [] }) + if (url.pathname.endsWith('/files/shortcut')) { + if (url.searchParams.get('alt') === 'media') { + aliasDownloads++ + return denied() + } + return aliasPresent ? json(alias()) : denied() + } + if (url.pathname.endsWith('/files/target')) { + expect(headers.get('X-Goog-Drive-Resource-Keys')).toBe('target/fixture-key') + if (targetMissing || (denyBob && headers.get('Authorization') === `Bearer ${ids.bobId}`)) + return denied() + if (url.searchParams.get('alt') === 'media') { + targetDownloads++ + return new Response(new Uint8Array(pdfBytes)) + } + return json(target()) + } + throw new Error('Unexpected fixture Drive endpoint') + } + async function sync() { + const result = await executeSync(ids.connectorId, { + fullSync: true, + billingAttribution: billing, + }) + expect(result.error).toBeUndefined() + expect(result.docsFailed).toBe(0) + return result + } + async function row(connectorId = ids.connectorId) { + const [value] = await db + .select() + .from(document) + .where(and(eq(document.connectorId, connectorId), eq(document.externalId, 'shortcut'))) + expect(value).toBeDefined() + return value! + } + async function chunks(documentId: string, userId = ids.aliceId) { + const result = await listKnowledgeChunks.execute({ + principal: principal(userId), + input: { knowledgeBaseId: ids.knowledgeBaseId, documentId }, + }) + return result.chunks.map((chunk) => chunk.content).join('\n') + } + async function search(userId: string) { + const result = await searchKnowledge.execute({ + principal: principal(userId), + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query: 'Orion', + searchMode: 'hybrid', + topK: 10, + }, + }) + return result.results.map((result) => result.documentId) + } + beforeAll(async () => { + fixture.storageRoot = mkdtempSync(path.join(tmpdir(), 'sim-drive-shortcuts-')) + await seedKnowledgeAclFixture(ids) + permittedTargetUser = ids.aliceId + await setPdf('Orion shortcut original content.') + billing = await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }) + await db + .update(knowledgeConnector) + .set({ + connectorType: 'google_drive', + sourceConfig: { folderId: 'root' }, + accessMode: 'workspace', + status: 'active', + syncLockToken: null, + }) + .where(eq(knowledgeConnector.id, ids.connectorId)) + vi.spyOn(connectorTokens, 'resolveConnectorAccessToken').mockResolvedValue({ + accessToken: 'fixture-admin', + }) + vi.stubGlobal('fetch', providerFetch) + }) + afterAll(async () => { + await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) + await db.delete(organization).where(eq(organization.id, ids.organizationId)) + await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + await rm(fixture.storageRoot, { recursive: true, force: true }) + vi.restoreAllMocks() + vi.unstubAllGlobals() + await db.$client.end() + }) + + it('recovers a failed shortcut row, indexes the real PDF, skips unchanged bytes, refreshes target-only edits and recovers a missing target', async () => { + const documentId = generateId() + await db.insert(document).values({ + id: documentId, + knowledgeBaseId: ids.knowledgeBaseId, + connectorId: ids.connectorId, + externalId: 'shortcut', + filename: 'Alias.pdf', + mimeType: 'text/plain', + fileUrl: '', + fileSize: 0, + processingStatus: 'failed', + processingError: 'Synthetic previous download failure', + }) + expect((await sync()).docsUpdated).toBe(1) + const original = await row() + expect(original.id).toBe(documentId) + expect(original.processingStatus).toBe('completed') + expect(await chunks(documentId)).toContain('Orion shortcut original content') + expect(await search(ids.aliceId)).toContain(documentId) + expect( + await downloadFileFromUrl(original.fileUrl, { userId: ids.aliceId, knowledgeAccess: 'user' }) + ).toEqual(pdfBytes) + const downloaded = targetDownloads + const embedded = fixture.embeddingCalls + expect((await sync()).docsUnchanged).toBe(1) + expect(targetDownloads).toBe(downloaded) + expect(fixture.embeddingCalls).toBe(embedded) + + revision++ + await setPdf('Orion shortcut revised content.') + expect((await sync()).docsUpdated).toBe(1) + expect(await chunks(documentId)).toContain('Orion shortcut revised content') + expect((await row()).contentHash).not.toBe(original.contentHash) + targetMissing = true + await sync() + expect((await row()).processingStatus).toBe('failed') + expect(await search(ids.aliceId)).not.toContain(documentId) + expect(await db.select().from(embedding).where(eq(embedding.documentId, documentId))).toEqual( + [] + ) + targetMissing = false + await sync() + expect((await row()).processingStatus).toBe('completed') + expect(await chunks(documentId)).toContain('Orion shortcut revised content') + expect(aliasDownloads).toBe(0) + }, 60000) + + it('persists shortcut and target ACL intersection and applies target-only permission changes without reembedding', async () => { + await db + .update(knowledgeConnector) + .set({ + accessMode: 'admin', + sourceConfig: { folderId: 'root', adminEmail: 'admin@fixture.test' }, + }) + .where(eq(knowledgeConnector.id, ids.connectorId)) + const indexedBefore = await row() + const priorEmbeddings = await db + .select({ id: embedding.id }) + .from(embedding) + .where(eq(embedding.documentId, indexedBefore.id)) + .orderBy(embedding.id) + const downloaded = targetDownloads + await sync() + const indexed = await row() + expect(indexed.aclRequirements).toHaveLength(2) + expect(indexed.aclRequirements).toEqual( + expect.arrayContaining([ + [`u:${ids.aliceId}@fixture.test`, `u:${ids.bobId}@fixture.test`].sort(), + [`u:${ids.aliceId}@fixture.test`], + ]) + ) + expect(await search(ids.aliceId)).toContain(indexed.id) + expect(await search(ids.bobId)).not.toContain(indexed.id) + await expect( + readKnowledgeDocument.execute({ + principal: principal(ids.bobId), + input: { knowledgeBaseId: ids.knowledgeBaseId, documentId: indexed.id }, + }) + ).rejects.toThrow() + permittedTargetUser = ids.bobId + await sync() + expect(await search(ids.aliceId)).not.toContain(indexed.id) + expect(await search(ids.bobId)).toContain(indexed.id) + expect( + await db + .select({ id: embedding.id }) + .from(embedding) + .where(eq(embedding.documentId, indexed.id)) + .orderBy(embedding.id) + ).toEqual(priorEmbeddings) + expect(targetDownloads).toBe(downloaded) + }, 60000) + + it('revokes and restores member search access when only target access changes and the provider feed is empty', async () => { + enrolled = await seedKnowledgeMemberFixture(ids) + await db + .update(knowledgeConnector) + .set({ memberSyncStatus: 'idle', memberSyncLockToken: null }) + .where(eq(knowledgeConnector.id, enrolled.connectorId)) + vi.spyOn(memberAccess, 'mintKnowledgeConnectorMemberToken').mockImplementation( + async ({ credentialId }) => ({ + accessToken: enrolled.members.find((member) => member.credentialId === credentialId)! + .userId, + refreshed: false, + }) + ) + await memberAccess.grantKnowledgeConnectorCredentialAccess( + { + workspaceId: ids.workspaceId, + connectorId: enrolled.connectorId, + credentialGroupId: enrolled.groupId, + credentialGroupOptionId: enrolled.optionId, + }, + ids.aliceId + ) + const syncMembers = async () => { + await db + .update(knowledgeConnectorMember) + .set({ nextAttemptAt: new Date(0) }) + .where(eq(knowledgeConnectorMember.connectorId, enrolled.connectorId)) + const result = await executeMemberSync(enrolled.connectorId, { billingAttribution: billing }) + expect(result.error).toBeUndefined() + return result + } + await syncMembers() + const indexed = await row(enrolled.connectorId) + expect(indexed.processingStatus).toBe('completed') + expect(await search(ids.bobId)).toContain(indexed.id) + const downloaded = targetDownloads + denyBob = true + await syncMembers() + expect(await search(ids.bobId)).not.toContain(indexed.id) + expect(await search(ids.aliceId)).toContain(indexed.id) + expect(targetDownloads).toBe(downloaded) + denyBob = false + await syncMembers() + expect(await search(ids.bobId)).toContain(indexed.id) + expect(targetDownloads).toBe(downloaded) + revision++ + await setPdf('Orion member shortcut target edit.') + await syncMembers() + expect(await chunks(indexed.id)).toContain('Orion member shortcut target edit') + expect(targetDownloads).toBe(downloaded + 1) + }, 60000) +}) diff --git a/apps/sim/lib/knowledge/connectors/connector-error.test.ts b/apps/sim/lib/knowledge/connectors/connector-error.test.ts index 843395c53a1..a1d52f97540 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.test.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.test.ts @@ -2,6 +2,7 @@ import { DrizzleQueryError } from 'drizzle-orm/errors' import { describe, expect, it } from 'vitest' import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' +import { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' describe('connector failure diagnostics', () => { it('retains the SQLSTATE while discarding SQL, bound values and driver detail', () => { @@ -68,6 +69,22 @@ describe('connector failure diagnostics', () => { expect(JSON.stringify(diagnostic)).not.toContain('private') }) + it.each([ + ['fileNotDownloadable', 'request_rejected'], + ['fileNotExportable', 'request_rejected'], + ['exportSizeLimitExceeded', 'request_rejected'], + ['domainPolicy', 'request_rejected'], + ['userRateLimitExceeded', 'rate_limit'], + ['dailyLimitExceeded', 'rate_limit'], + ['insufficientFilePermissions', 'authorization'], + ])('preserves provider classification for HTTP 403 %s', (reason, category) => { + const error = new Error('private wrapper', { cause: new GoogleDriveApiError(403, [reason]) }) + expect(getConnectorFailureDiagnostic(error)).toMatchObject({ status: 403, category }) + expect(JSON.stringify(getConnectorFailureDiagnostic(error))).not.toContain('private') + if (category !== 'authorization') + expect(getConnectorFailureDiagnostic(error)?.message).not.toContain('access was denied') + }) + it('does not infer status or permanence from a free-form message', () => { expect(getConnectorFailureDiagnostic(new Error('HTTP 403 permission denied'))).toBeNull() expect( diff --git a/apps/sim/lib/knowledge/connectors/connector-error.ts b/apps/sim/lib/knowledge/connectors/connector-error.ts index 97e0623073e..7f8e4ac5179 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.ts @@ -1,15 +1,12 @@ import { findCause, getPostgresErrorCode } from '@sim/utils/errors' import { DrizzleQueryError } from 'drizzle-orm/errors' +import { + ConnectorSourceError, + type ConnectorSourceFailureCategory, +} from '@/connectors/source-error' export interface ConnectorFailureDiagnostic { - category: - | 'database' - | 'authorization' - | 'source_unavailable' - | 'request_rejected' - | 'rate_limit' - | 'provider_unavailable' - | 'transport' + category: 'database' | ConnectorSourceFailureCategory | 'transport' message: string status?: number code?: string @@ -74,29 +71,29 @@ export function getConnectorFailureDiagnostic(error: unknown): ConnectorFailureD ) if (!httpError) return null const { status } = httpError - if (status === 401 || status === 403) { + const category = httpError instanceof ConnectorSourceError ? httpError.category : undefined + if (category === 'authorization' || (!category && (status === 401 || status === 403))) { return { category: 'authorization', status, message: `Source content access was denied (HTTP ${status}). Check the connector account's file access and download permissions.`, } } - if (status === 404 || status === 410) { + if (category === 'source_unavailable' || (!category && (status === 404 || status === 410))) { return { category: 'source_unavailable', status, message: `Source content is unavailable (HTTP ${status}). It may have moved, been removed, or lost sharing access.`, } } - if (status === 429) { + if (category === 'rate_limit' || (!category && status === 429)) { return { category: 'rate_limit', status, - message: - 'Source requests are rate limited (HTTP 429). The connector will retry after backoff.', + message: `Source request quota or rate limit was exceeded (HTTP ${status}). The connector will retry after backoff.`, } } - if (status >= 500 || status === 408) { + if (category === 'provider_unavailable' || (!category && (status >= 500 || status === 408))) { return { category: 'provider_unavailable', status,