Skip to content

Commit 4d39b0c

Browse files
waleedlatif1waleedlatif1
authored
feat(connectors): use resource selectors for KB connector config (#5116)
* feat(connectors): use resource selectors for KB connector config Replace raw ID text inputs with selector pickers (canonical selector + manual-input pairs) across Google Drive/Docs/Forms/Sheets, Notion, Monday, and Webflow KB connectors, so users pick folders/spreadsheets/pages/boards/ collections instead of pasting IDs — matching the workflow blocks. - Add multi-select where the sync handler supports it (Drive/Docs/Forms folders, Monday boards, Webflow collections) via parseMultiValue - Add shared escapeDriveQueryValue/buildDriveParentsClause helpers for safe multi-folder Drive queries - Add ConnectorConfigField.mimeType, plumbed into the selector context - Fix Webflow listingCapped not set on maxItems truncation (deletion- reconciliation data-loss safety) Fully backward compatible: legacy single-string IDs and CSV both normalize via parseMultiValue; resolved canonical keys are unchanged. * fix(webflow): set listingCapped on within-page maxItems truncation When a collection's items fit in a single API page but maxItems cuts the list within that page, neither hasMoreInCollection nor hasMoreCollections is true, so listingCapped was not set and the sync engine could hard-delete still-existing documents. Add the within-page drop signal to the guard.
1 parent ea505f0 commit 4d39b0c

14 files changed

Lines changed: 280 additions & 108 deletions

File tree

‎apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ export function ConnectorSelectorField({
3838
const context = useMemo<SelectorContext>(() => {
3939
const ctx: SelectorContext = {}
4040
if (credentialId) ctx.oauthCredential = credentialId
41+
if (field.mimeType) ctx.mimeType = field.mimeType
4142

4243
for (const depFieldId of getDependsOnFields(field.dependsOn)) {
4344
const depField = configFields.find((f) => f.id === depFieldId)
@@ -49,7 +50,7 @@ export function ConnectorSelectorField({
4950
}
5051

5152
return ctx
52-
}, [credentialId, field.dependsOn, sourceConfig, configFields, canonicalModes])
53+
}, [credentialId, field.mimeType, field.dependsOn, sourceConfig, configFields, canonicalModes])
5354

5455
const depsResolved = useMemo(() => {
5556
if (!field.dependsOn) return true

‎apps/sim/connectors/google-docs/google-docs.ts‎

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,12 @@ import { toError } from '@sim/utils/errors'
33
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
44
import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta'
55
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
6-
import { joinTagArray, parseTagDate } from '@/connectors/utils'
6+
import {
7+
buildDriveParentsClause,
8+
joinTagArray,
9+
parseMultiValue,
10+
parseTagDate,
11+
} from '@/connectors/utils'
712

813
const logger = createLogger('GoogleDocsConnector')
914

@@ -152,10 +157,8 @@ function fileToStub(file: DriveFile): ExternalDocument {
152157
function buildQuery(sourceConfig: Record<string, unknown>): string {
153158
const parts: string[] = ['trashed = false', "mimeType = 'application/vnd.google-apps.document'"]
154159

155-
const folderId = sourceConfig.folderId as string | undefined
156-
if (folderId?.trim()) {
157-
parts.push(`'${folderId.trim().replace(/\\/g, '\\\\').replace(/'/g, "\\'")}' in parents`)
158-
}
160+
const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId))
161+
if (parentsClause) parts.push(parentsClause)
159162

160163
return parts.join(' and ')
161164
}
@@ -298,38 +301,46 @@ export const googleDocsConnector: ConnectorConfig = {
298301
accessToken: string,
299302
sourceConfig: Record<string, unknown>
300303
): Promise<{ valid: boolean; error?: string }> => {
301-
const folderId = sourceConfig.folderId as string | undefined
304+
const folderIds = parseMultiValue(sourceConfig.folderId)
302305
const maxDocs = sourceConfig.maxDocs as string | undefined
303306

304307
if (maxDocs && (Number.isNaN(Number(maxDocs)) || Number(maxDocs) <= 0)) {
305308
return { valid: false, error: 'Max documents must be a positive number' }
306309
}
307310

308311
try {
309-
if (folderId?.trim()) {
310-
const url = `https://www.googleapis.com/drive/v3/files/${folderId.trim()}?fields=id,name,mimeType&supportsAllDrives=true`
311-
const response = await fetchWithRetry(
312-
url,
313-
{
314-
method: 'GET',
315-
headers: {
316-
Authorization: `Bearer ${accessToken}`,
317-
Accept: 'application/json',
312+
if (folderIds.length > 0) {
313+
for (const folderId of folderIds) {
314+
const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
315+
const response = await fetchWithRetry(
316+
url,
317+
{
318+
method: 'GET',
319+
headers: {
320+
Authorization: `Bearer ${accessToken}`,
321+
Accept: 'application/json',
322+
},
318323
},
319-
},
320-
VALIDATE_RETRY_OPTIONS
321-
)
322-
323-
if (!response.ok) {
324-
if (response.status === 404) {
325-
return { valid: false, error: 'Folder not found. Check the folder ID and permissions.' }
324+
VALIDATE_RETRY_OPTIONS
325+
)
326+
327+
if (!response.ok) {
328+
if (response.status === 404) {
329+
return {
330+
valid: false,
331+
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
332+
}
333+
}
334+
return {
335+
valid: false,
336+
error: `Failed to access folder "${folderId}": ${response.status}`,
337+
}
326338
}
327-
return { valid: false, error: `Failed to access folder: ${response.status}` }
328-
}
329339

330-
const folder = await response.json()
331-
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
332-
return { valid: false, error: 'The provided ID is not a folder' }
340+
const folder = await response.json()
341+
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
342+
return { valid: false, error: `"${folderId}" is not a folder` }
343+
}
333344
}
334345
} else {
335346
const url =

‎apps/sim/connectors/google-docs/meta.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,26 @@ export const googleDocsConnectorMeta: ConnectorMeta = {
1515
},
1616

1717
configFields: [
18+
{
19+
id: 'folderSelector',
20+
title: 'Folders',
21+
type: 'selector',
22+
selectorKey: 'google.drive',
23+
mimeType: 'application/vnd.google-apps.folder',
24+
canonicalParamId: 'folderId',
25+
mode: 'basic',
26+
multi: true,
27+
placeholder: 'Select one or more folders (optional)',
28+
required: false,
29+
},
1830
{
1931
id: 'folderId',
20-
title: 'Folder ID',
32+
title: 'Folder IDs',
2133
type: 'short-input',
22-
placeholder: 'e.g. 1aBcDeFgHiJkLmNoPqRsTuVwXyZ (optional)',
34+
canonicalParamId: 'folderId',
35+
mode: 'advanced',
36+
multi: true,
37+
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
2338
required: false,
2439
},
2540
{

‎apps/sim/connectors/google-drive/google-drive.ts‎

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@ import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/document
44
import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta'
55
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
66
import {
7+
buildDriveParentsClause,
78
CONNECTOR_MAX_FILE_BYTES,
89
ConnectorFileTooLargeError,
910
htmlToPlainText,
1011
joinTagArray,
1112
markSkipped,
13+
parseMultiValue,
1214
parseTagDate,
1315
readBodyWithLimit,
1416
sizeLimitSkipReason,
@@ -137,10 +139,8 @@ interface DriveFile {
137139
function buildQuery(sourceConfig: Record<string, unknown>): string {
138140
const parts: string[] = ['trashed = false']
139141

140-
const folderId = sourceConfig.folderId as string | undefined
141-
if (folderId?.trim()) {
142-
parts.push(`'${folderId.trim().replace(/\\/g, '\\\\').replace(/'/g, "\\'")}' in parents`)
143-
}
142+
const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId))
143+
if (parentsClause) parts.push(parentsClause)
144144

145145
const fileType = (sourceConfig.fileType as string) || 'all'
146146
switch (fileType) {
@@ -324,7 +324,7 @@ export const googleDriveConnector: ConnectorConfig = {
324324
accessToken: string,
325325
sourceConfig: Record<string, unknown>
326326
): Promise<{ valid: boolean; error?: string }> => {
327-
const folderId = sourceConfig.folderId as string | undefined
327+
const folderIds = parseMultiValue(sourceConfig.folderId)
328328
const maxFiles = sourceConfig.maxFiles as string | undefined
329329

330330
if (maxFiles && (Number.isNaN(Number(maxFiles)) || Number(maxFiles) <= 0)) {
@@ -333,31 +333,39 @@ export const googleDriveConnector: ConnectorConfig = {
333333

334334
// Verify access to Drive API
335335
try {
336-
if (folderId?.trim()) {
337-
// Verify the folder exists and is accessible
338-
const url = `https://www.googleapis.com/drive/v3/files/${folderId.trim()}?fields=id,name,mimeType&supportsAllDrives=true`
339-
const response = await fetchWithRetry(
340-
url,
341-
{
342-
method: 'GET',
343-
headers: {
344-
Authorization: `Bearer ${accessToken}`,
345-
Accept: 'application/json',
336+
if (folderIds.length > 0) {
337+
// Verify each folder exists, is accessible, and is actually a folder
338+
for (const folderId of folderIds) {
339+
const url = `https://www.googleapis.com/drive/v3/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
340+
const response = await fetchWithRetry(
341+
url,
342+
{
343+
method: 'GET',
344+
headers: {
345+
Authorization: `Bearer ${accessToken}`,
346+
Accept: 'application/json',
347+
},
346348
},
347-
},
348-
VALIDATE_RETRY_OPTIONS
349-
)
350-
351-
if (!response.ok) {
352-
if (response.status === 404) {
353-
return { valid: false, error: 'Folder not found. Check the folder ID and permissions.' }
349+
VALIDATE_RETRY_OPTIONS
350+
)
351+
352+
if (!response.ok) {
353+
if (response.status === 404) {
354+
return {
355+
valid: false,
356+
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
357+
}
358+
}
359+
return {
360+
valid: false,
361+
error: `Failed to access folder "${folderId}": ${response.status}`,
362+
}
354363
}
355-
return { valid: false, error: `Failed to access folder: ${response.status}` }
356-
}
357364

358-
const folder = await response.json()
359-
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
360-
return { valid: false, error: 'The provided ID is not a folder' }
365+
const folder = await response.json()
366+
if (folder.mimeType !== 'application/vnd.google-apps.folder') {
367+
return { valid: false, error: `"${folderId}" is not a folder` }
368+
}
361369
}
362370
} else {
363371
// Verify basic Drive access by listing one file

‎apps/sim/connectors/google-drive/meta.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,26 @@ export const googleDriveConnectorMeta: ConnectorMeta = {
1515
},
1616

1717
configFields: [
18+
{
19+
id: 'folderSelector',
20+
title: 'Folders',
21+
type: 'selector',
22+
selectorKey: 'google.drive',
23+
mimeType: 'application/vnd.google-apps.folder',
24+
canonicalParamId: 'folderId',
25+
mode: 'basic',
26+
multi: true,
27+
placeholder: 'Select one or more folders (optional)',
28+
required: false,
29+
},
1830
{
1931
id: 'folderId',
20-
title: 'Folder ID',
32+
title: 'Folder IDs',
2133
type: 'short-input',
22-
placeholder: 'e.g. 1aBcDeFgHiJkLmNoPqRsTuVwXyZ (optional)',
34+
canonicalParamId: 'folderId',
35+
mode: 'advanced',
36+
multi: true,
37+
placeholder: 'e.g. 1aBcDeFg…, 2cDeFgHi… (comma-separated for multiple)',
2338
required: false,
2439
},
2540
{

‎apps/sim/connectors/google-forms/google-forms.ts‎

Lines changed: 45 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,12 @@ import { getErrorMessage, toError } from '@sim/utils/errors'
33
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
44
import { googleFormsConnectorMeta, MAX_RESPONSES_PER_FORM } from '@/connectors/google-forms/meta'
55
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
6-
import { joinTagArray, parseTagDate } from '@/connectors/utils'
6+
import {
7+
buildDriveParentsClause,
8+
joinTagArray,
9+
parseMultiValue,
10+
parseTagDate,
11+
} from '@/connectors/utils'
712

813
const logger = createLogger('GoogleFormsConnector')
914

@@ -448,16 +453,14 @@ function renderFormDocument(form: FormStructure, responses: FormResponse[]): str
448453
}
449454

450455
/**
451-
* Builds the Drive `q` query that selects form files, optionally scoped to a
452-
* folder. Single quotes and backslashes in the folder ID are escaped to prevent
453-
* query injection.
456+
* Builds the Drive `q` query that selects form files, optionally scoped to one
457+
* or more folders. Single quotes and backslashes in folder IDs are escaped to
458+
* prevent query injection.
454459
*/
455-
function buildDriveQuery(folderId?: string): string {
460+
function buildDriveQuery(folderIds: string[]): string {
456461
const parts = ['trashed = false', `mimeType = '${FORM_MIME_TYPE}'`]
457-
if (folderId?.trim()) {
458-
const escaped = folderId.trim().replace(/\\/g, '\\\\').replace(/'/g, "\\'")
459-
parts.push(`'${escaped}' in parents`)
460-
}
462+
const parentsClause = buildDriveParentsClause(folderIds)
463+
if (parentsClause) parts.push(parentsClause)
461464
return parts.join(' and ')
462465
}
463466

@@ -479,9 +482,9 @@ export const googleFormsConnector: ConnectorConfig = {
479482
return { documents: [], hasMore: false }
480483
}
481484

482-
const folderId = sourceConfig.folderId as string | undefined
485+
const folderIds = parseMultiValue(sourceConfig.folderId)
483486
const queryParams = new URLSearchParams({
484-
q: buildDriveQuery(folderId),
487+
q: buildDriveQuery(folderIds),
485488
pageSize: String(DRIVE_PAGE_SIZE),
486489
orderBy: 'modifiedTime desc',
487490
fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners)',
@@ -493,7 +496,7 @@ export const googleFormsConnector: ConnectorConfig = {
493496
const url = `${DRIVE_API_BASE}/files?${queryParams.toString()}`
494497

495498
logger.info('Listing Google Forms', {
496-
folderId: folderId?.trim() || 'all',
499+
folderId: folderIds.length > 0 ? folderIds.join(',') : 'all',
497500
contentScope,
498501
cursor: cursor ?? 'initial',
499502
})
@@ -667,7 +670,7 @@ export const googleFormsConnector: ConnectorConfig = {
667670
accessToken: string,
668671
sourceConfig: Record<string, unknown>
669672
): Promise<{ valid: boolean; error?: string }> => {
670-
const folderId = sourceConfig.folderId as string | undefined
673+
const folderIds = parseMultiValue(sourceConfig.folderId)
671674
const maxForms = sourceConfig.maxForms as string | undefined
672675
const maxResponsesPerForm = sourceConfig.maxResponsesPerForm as string | undefined
673676

@@ -683,30 +686,38 @@ export const googleFormsConnector: ConnectorConfig = {
683686
}
684687

685688
try {
686-
if (folderId?.trim()) {
687-
const url = `${DRIVE_API_BASE}/files/${encodeURIComponent(folderId.trim())}?fields=id,name,mimeType&supportsAllDrives=true`
688-
const response = await fetchWithRetry(
689-
url,
690-
{
691-
method: 'GET',
692-
headers: {
693-
Authorization: `Bearer ${accessToken}`,
694-
Accept: 'application/json',
689+
if (folderIds.length > 0) {
690+
for (const folderId of folderIds) {
691+
const url = `${DRIVE_API_BASE}/files/${encodeURIComponent(folderId)}?fields=id,name,mimeType&supportsAllDrives=true`
692+
const response = await fetchWithRetry(
693+
url,
694+
{
695+
method: 'GET',
696+
headers: {
697+
Authorization: `Bearer ${accessToken}`,
698+
Accept: 'application/json',
699+
},
695700
},
696-
},
697-
VALIDATE_RETRY_OPTIONS
698-
)
699-
700-
if (!response.ok) {
701-
if (response.status === 404) {
702-
return { valid: false, error: 'Folder not found. Check the folder ID and permissions.' }
701+
VALIDATE_RETRY_OPTIONS
702+
)
703+
704+
if (!response.ok) {
705+
if (response.status === 404) {
706+
return {
707+
valid: false,
708+
error: `Folder "${folderId}" not found. Check the folder ID and permissions.`,
709+
}
710+
}
711+
return {
712+
valid: false,
713+
error: `Failed to access folder "${folderId}": ${response.status}`,
714+
}
703715
}
704-
return { valid: false, error: `Failed to access folder: ${response.status}` }
705-
}
706716

707-
const folder = await response.json()
708-
if (folder.mimeType !== FOLDER_MIME_TYPE) {
709-
return { valid: false, error: 'The provided ID is not a folder' }
717+
const folder = await response.json()
718+
if (folder.mimeType !== FOLDER_MIME_TYPE) {
719+
return { valid: false, error: `"${folderId}" is not a folder` }
720+
}
710721
}
711722
} else {
712723
const url = `${DRIVE_API_BASE}/files?pageSize=1&q=${encodeURIComponent(`mimeType = '${FORM_MIME_TYPE}'`)}&fields=files(id)&supportsAllDrives=true&includeItemsFromAllDrives=true`

0 commit comments

Comments
 (0)