diff --git a/backend/src/serverless/integrations/usecases/gitlab/getProjects.ts b/backend/src/serverless/integrations/usecases/gitlab/getProjects.ts index 41f05205e5..aba3524e5a 100644 --- a/backend/src/serverless/integrations/usecases/gitlab/getProjects.ts +++ b/backend/src/serverless/integrations/usecases/gitlab/getProjects.ts @@ -23,37 +23,48 @@ export async function fetchAllGitlabGroups(accessToken: string) { })) } -export async function fetchGitlabGroupProjects(accessToken: string, groups: any[]) { - const groupProjects = {} +async function fetchProjectsForGroup(accessToken: string, group: any) { + const projects = [] + let page = 1 + let hasMorePages = true - for (const group of groups) { - const projects = [] - let page = 1 - let hasMorePages = true + while (hasMorePages) { + const response = await axios.get(`https://gitlab.com/api/v4/groups/${group.id}/projects`, { + headers: { Authorization: `Bearer ${accessToken}` }, + params: { page, per_page: 100, archived: false }, + }) + projects.push(...response.data) + hasMorePages = response.headers['x-next-page'] !== '' + page++ + } - while (hasMorePages) { - const response = await axios.get(`https://gitlab.com/api/v4/groups/${group.id}/projects`, { - headers: { Authorization: `Bearer ${accessToken}` }, - params: { page, per_page: 100, archived: false }, - }) - projects.push(...response.data) - hasMorePages = response.headers['x-next-page'] !== '' - page++ - } + return projects.map((project) => ({ + groupId: group.id, + groupName: group.name, + groupPath: group.path, + id: project.id, + name: project.name, + path_with_namespace: project.path_with_namespace, + enabled: false, + forkedFrom: project?.forked_from_project?.web_url || null, + })) +} - groupProjects[group.id] = projects.map((project) => ({ - groupId: group.id, - groupName: group.name, - groupPath: group.path, - id: project.id, - name: project.name, - path_with_namespace: project.path_with_namespace, - enabled: false, - forkedFrom: project?.forked_from_project?.web_url || null, - })) +export async function fetchGitlabGroupProjects(accessToken: string, groups: any[]) { + const CONCURRENCY = 10 + const groupProjects: Record = {} + + for (let i = 0; i < groups.length; i += CONCURRENCY) { + const batch = groups.slice(i, i + CONCURRENCY) + const results = await Promise.all( + batch.map((group) => fetchProjectsForGroup(accessToken, group)), + ) + batch.forEach((group, idx) => { + groupProjects[group.id] = results[idx] + }) } - return groupProjects as Record + return groupProjects } export async function fetchGitlabUserProjects(accessToken: string, userId: number) { diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 8a2a149679..2ff1365a15 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -2774,6 +2774,18 @@ export default class IntegrationService { await SequelizeRepository.commitTransaction(transaction) } catch (err) { + this.options.log.error( + { + errMessage: err?.message, + errName: err?.name, + errStack: err?.stack, + gitlabStatus: err?.response?.status, + gitlabError: err?.response?.data, + gitlabUrl: err?.config?.url, + gitlabMethod: err?.config?.method, + }, + 'gitlabConnect failed', + ) await SequelizeRepository.rollbackTransaction(transaction) throw err }