feat(gitlab): add projectQuery support for dynamic project discovery

packages/backend/src/gitlab.test.ts

@@ -1,5 +1,52 @@
-import { expect, test } from 'vitest';
-import { shouldExcludeProject } from './gitlab';
+import { expect, test, vi, describe, beforeEach } from 'vitest';
+import { shouldExcludeProject, parseQuery, getGitLabReposFromConfig } from './gitlab';
+import { ProjectSchema } from '@gitbeaker/rest';
+
+// Mock dependencies
+const mockGitlabAll = vi.fn();
+
+vi.mock('@gitbeaker/rest', () => {
+    return {
+        Gitlab: vi.fn().mockImplementation(() => ({
+            Projects: {
+                all: mockGitlabAll,
+            }
+        })),
+    };
+});
+
+vi.mock('@sourcebot/shared', async () => ({
+    getTokenFromConfig: vi.fn(),
+    createLogger: () => ({
+        debug: vi.fn(),
+        info: vi.fn(),
+        warn: vi.fn(),
+        error: vi.fn(),
+        warning: vi.fn(),
+    }),
+    env: {
+        GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS: 10,
+    }
+}));
+
+vi.mock('./connectionUtils', () => ({
+    processPromiseResults: (results: any[]) => {
+        const validItems = results
+            .filter((r) => r.status === 'fulfilled' && r.value.type === 'valid')
+            .flatMap((r) => r.value.data);
+        return { validItems, warnings: [] };
+    },
+    throwIfAnyFailed: vi.fn(),
+}));
+
+vi.mock('./utils', () => ({
+    measure: async (fn: () => any) => {
+        const data = await fn();
+        return { durationMs: 0, data };
+    },
+    fetchWithRetry: async (fn: () => any) => fn(),
+}));
+import { shouldExcludeProject, parseQuery } from './gitlab';
 import { ProjectSchema } from '@gitbeaker/rest';
 
 
@@ -68,3 +115,63 @@ test('shouldExcludeProject returns false when exclude.userOwnedProjects is true
         exclude: { userOwnedProjects: true },
     })).toBe(false);
 });
+
+test('parseQuery correctly parses query strings', () => {
+    expect(parseQuery('projects?include_subgroups=true&archived=false&id=123')).toEqual({
+        include_subgroups: true,
+        archived: false,
+        id: 123
+    });
+    expect(parseQuery('projects?search=foo')).toEqual({
+        search: 'foo'
+    });
+    expect(parseQuery('groups/1/projects?simple=true')).toEqual({
+        simple: true
+    });
+});
+
+describe('getGitLabReposFromConfig', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+
+    test('fetches projects using projectQuery and deduplicates results', async () => {
+        const mockProjects1 = [
+            { id: 1, path_with_namespace: 'group1/project1', name: 'project1' },
+            { id: 2, path_with_namespace: 'group1/project2', name: 'project2' },
+        ];
+        const mockProjects2 = [
+            { id: 2, path_with_namespace: 'group1/project2', name: 'project2' }, // Duplicate
+            { id: 3, path_with_namespace: 'group2/project3', name: 'project3' },
+        ];
+
+        mockGitlabAll.mockResolvedValueOnce(mockProjects1);
+        mockGitlabAll.mockResolvedValueOnce(mockProjects2);
+
+        const config = {
+            type: 'gitlab' as const,
+            projectQuery: [
+                'groups/group1/projects?include_subgroups=true',
+                'projects?topic=devops'
+            ]
+        };
+
+        const result = await getGitLabReposFromConfig(config);
+
+        // Verify API calls
+        expect(mockGitlabAll).toHaveBeenCalledTimes(2);
+        expect(mockGitlabAll).toHaveBeenCalledWith(expect.objectContaining({
+            perPage: 100,
+            include_subgroups: true
+        }));
+        expect(mockGitlabAll).toHaveBeenCalledWith(expect.objectContaining({
+            perPage: 100,
+            topic: 'devops'
+        }));
+
+        // Verify deduplication
+        expect(result.repos).toHaveLength(3);
+        const projectIds = result.repos.map((p: any) => p.id).sort();
+        expect(projectIds).toEqual([1, 2, 3]);
+    });
+});

packages/backend/src/gitlab.ts

@@ -41,8 +41,8 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
     const token = config.token ?
         await getTokenFromConfig(config.token) :
         hostname === GITLAB_CLOUD_HOSTNAME ?
-        env.FALLBACK_GITLAB_CLOUD_TOKEN :
-        undefined;
+            env.FALLBACK_GITLAB_CLOUD_TOKEN :
+            undefined;
 
     const api = await createGitLabFromPersonalAccessToken({
         token,
@@ -189,13 +189,58 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
             }
         }));
 
+        const { validItems: validRepos, warnings } = processPromiseResults(results);
+        allRepos = allRepos.concat(validRepos);
+        allWarnings = allWarnings.concat(warnings);
+    }
+
+    if (config.projectQuery) {
+        const results = await Promise.allSettled(config.projectQuery.map(async (query) => {
+            try {
+                logger.debug(`Fetching projects for query ${query}...`);
+                const { durationMs, data } = await measure(async () => {
+                    const fetchFn = () => api.Projects.all({
+                        perPage: 100,
+                        ...parseQuery(query),
+                    } as any);
+                    return fetchWithRetry(fetchFn, `query ${query}`, logger);
+                });
+                logger.debug(`Found ${data.length} projects for query ${query} in ${durationMs}ms.`);
+                return {
+                    type: 'valid' as const,
+                    data
+                };
+            } catch (e: any) {
+                Sentry.captureException(e);
+                logger.error(`Failed to fetch projects for query ${query}.`, e);
+
+                const status = e?.cause?.response?.status;
+                if (status !== undefined) {
+                    const warning = `GitLab API returned ${status}`
+                    logger.warning(warning);
+                    return {
+                        type: 'warning' as const,
+                        warning
+                    }
+                }
+
+                logger.error("No API response status returned");
+                throw e;
+            }
+        }));
+
         throwIfAnyFailed(results);
         const { validItems: validRepos, warnings } = processPromiseResults(results);
         allRepos = allRepos.concat(validRepos);
         allWarnings = allWarnings.concat(warnings);
     }
 
     let repos = allRepos
+        .filter((project, index, self) =>
+            index === self.findIndex((t) => (
+                t.id === project.id
+            ))
+        )
         .filter((project) => {
             const isExcluded = shouldExcludeProject({
                 project,
@@ -207,7 +252,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
 
             return !isExcluded;
         });
-        
+
     logger.debug(`Found ${repos.length} total repositories.`);
 
     return {
@@ -300,6 +345,23 @@ export const getProjectMembers = async (projectId: string, api: InstanceType<typ
     }
 }
 
+export const parseQuery = (query: string) => {
+    const params = new URLSearchParams(query.split('?')[1]);
+    const result: Record<string, string | boolean | number> = {};
+    for (const [key, value] of params.entries()) {
+        if (value === 'true') {
+            result[key] = true;
+        } else if (value === 'false') {
+            result[key] = false;
+        } else if (!isNaN(Number(value))) {
+            result[key] = Number(value);
+        } else {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+
 export const getProjectsForAuthenticatedUser = async (visibility: 'private' | 'internal' | 'public' | 'all' = 'all', api: InstanceType<typeof Gitlab>) => {
     try {
         const fetchFn = () => api.Projects.all({

schemas/v3/gitlab.json

@@ -65,6 +65,19 @@
             ],
             "description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/"
         },
+        "projectQuery": {
+            "type": "array",
+            "items": {
+                "type": "string"
+            },
+            "examples": [
+                [
+                    "groups/group1/projects?include_subgroups=true",
+                    "projects?topic=devops"
+                ]
+            ],
+            "description": "List of GitLab API query paths to fetch projects from. Each string should be a path relative to the GitLab API root (e.g. `groups/my-group/projects`)."
+        },
         "topics": {
             "type": "array",
             "items": {