main.go

package main

import (
	"bytes"
	"context"
	"encoding/csv"
	"flag"
	"fmt"
	"math"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"sync"
	"time"

	"github.com/gofri/go-github-pagination/githubpagination"
	"github.com/google/go-github/v74/github"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/go-hclog"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/xanzy/go-gitlab"
)

const (
	dateFormat             = time.RFC3339
	defaultGithubDomain    = "github.com"
	defaultGitlabDomain    = "gitlab.com"
	defaultImagesRepoName = "gl-imgs"
	defaultImagesRepoRef  = "main"
)

var loop, report bool
var deleteExistingRepos, enablePullRequests, renameMasterToMain, skipInvalidMergeRequests, trimGithubBranches, skipExistingClosedOrMergedMergeRequests, skipMigratingComments, onlyMigratePullRequests, onlyMigrateComments bool
var githubDomain, githubRepo, githubToken, githubUser, gitlabDomain, gitlabProject, gitlabToken, projectsCsvPath, renameTrunkBranch string
var imagesRepoName, imagesRepoRef string
var mergeRequestsAge int
var mergeRequestsFromID int

var (
	cache                     *objectCache
	errCount                  int
	logger                    hclog.Logger
	gh                        *github.Client
	gl                        *gitlab.Client
	maxConcurrency            int
	maxConcurrencyForComments int
	version                   = "development"
)

type Project = []string

type Report struct {
	GroupName          string
	ProjectName        string
	MergeRequestsCount int
}

type GitHubError struct {
	Message          string
	DocumentationURL string `json:"documentation_url"`
}

func main() {
	var err error

	// Bypass pre-emptive rate limit checks in the GitHub client, as we will handle these via go-retryablehttp
	valueCtx := context.WithValue(context.Background(), github.BypassRateLimitCheck, true)

	// Assign a Done channel so we can abort on Ctrl-c
	ctx, cancel := context.WithCancel(valueCtx)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	defer func() {
		signal.Stop(c)
		cancel()
	}()
	go func() {
		select {
		case <-c:
			cancel()
		case <-ctx.Done():
		}
	}()

	logger = hclog.New(&hclog.LoggerOptions{
		Name:  "gitlab-migrator",
		Level: hclog.LevelFromString(os.Getenv("LOG_LEVEL")),
	})

	cache = newObjectCache()

	var showVersion bool
	fmt.Printf(fmt.Sprintf("gitlab-migrator %s\n", version))

	flag.BoolVar(&loop, "loop", false, "continue migrating until canceled")
	flag.BoolVar(&report, "report", false, "report on primitives to be migrated instead of beginning migration")

	flag.BoolVar(&deleteExistingRepos, "delete-existing-repos", false, "whether existing repositories should be deleted before migrating")
	flag.BoolVar(&enablePullRequests, "migrate-pull-requests", false, "whether pull requests should be migrated")
	flag.BoolVar(&renameMasterToMain, "rename-master-to-main", false, "rename master branch to main and update pull requests (incompatible with -rename-trunk-branch)")
	flag.BoolVar(&skipInvalidMergeRequests, "skip-invalid-merge-requests", false, "when true, will log and skip invalid merge requests instead of raising an error")
	flag.BoolVar(&trimGithubBranches, "trim-branches-on-github", false, "when true, will delete any branches on GitHub that are no longer present in GitLab")
	flag.BoolVar(&skipExistingClosedOrMergedMergeRequests, "skip-existing-closed-or-merged-merge-requests", false,
		"when true, will skip migrating closed/merged merge requests that already have corresponding closed pull requests on GitHub - used only when migrate-pull-requests or only-migrate-pull-requests is set, and only-migrate-comments is not set")
	flag.BoolVar(&skipMigratingComments, "skip-migrating-comments", false, "when true, will skip migrating comments - used only when migrate-pull-requests or only-migrate-pull-requests is set, and only-migrate-comments is not set")
	flag.BoolVar(&onlyMigratePullRequests, "only-migrate-pull-requests", false, "when true, will only migrate pull requests - this short-circuits much of the repo migration logic - used only when only-migrate-comments is not set")
	flag.BoolVar(&onlyMigrateComments, "only-migrate-comments", false, "when true, will only migrate comments - this short-circuits much of the repo/MR migration logic, and uses goroutines for parallelization")
	flag.BoolVar(&showVersion, "version", false, "output version information")

	flag.StringVar(&githubDomain, "github-domain", defaultGithubDomain, "specifies the GitHub domain to use")
	flag.StringVar(&githubRepo, "github-repo", "", "the GitHub repository to migrate to")
	flag.StringVar(&githubUser, "github-user", "", "specifies the GitHub user to use, who will author any migrated PRs (required)")
	flag.StringVar(&gitlabDomain, "gitlab-domain", defaultGitlabDomain, "specifies the GitLab domain to use")
	flag.StringVar(&gitlabProject, "gitlab-project", "", "the GitLab project to migrate")
	flag.StringVar(&projectsCsvPath, "projects-csv", "", "specifies the path to a CSV file describing projects to migrate (incompatible with -gitlab-project and -github-repo)")
	flag.IntVar(&mergeRequestsAge, "merge-requests-max-age", 0, "optional maximum age in days of merge requests to migrate")
	flag.IntVar(&mergeRequestsFromID, "merge-requests-from-id", 0, "optional merge request ID to start migrating from, inclusive")
	flag.StringVar(&renameTrunkBranch, "rename-trunk-branch", "", "specifies the new trunk branch name (incompatible with -rename-master-to-main)")
	flag.StringVar(&imagesRepoName, "images-repo-name", defaultImagesRepoName, "specifies the repository to use for GL images")
	flag.StringVar(&imagesRepoRef, "images-repo-ref", defaultImagesRepoRef, "specifies the commit SHA to use for GL images")

	flag.IntVar(&maxConcurrency, "max-concurrency", 4, "how many projects to migrate in parallel")
	flag.IntVar(&maxConcurrencyForComments, "max-concurrency-for-comments", 2, "how many merge request comments to migrate in parallel - used only when only-migrate-comments is set")

	flag.Parse()

	if showVersion {
		return
	}

	githubToken = os.Getenv("GITHUB_TOKEN")
	if githubToken == "" {
		logger.Error("missing environment variable", "name", "GITHUB_TOKEN")
		os.Exit(1)
	}

	gitlabToken = os.Getenv("GITLAB_TOKEN")
	if gitlabToken == "" {
		logger.Error("missing environment variable", "name", "GITLAB_TOKEN")
		os.Exit(1)
	}

	if githubUser == "" {
		githubUser = os.Getenv("GITHUB_USER")
	}

	if githubUser == "" {
		logger.Error("must specify GitHub user")
		os.Exit(1)
	}

	repoSpecifiedInline := githubRepo != "" && gitlabProject != ""
	if repoSpecifiedInline && projectsCsvPath != "" {
		logger.Error("cannot specify -projects-csv and either -github-repo or -gitlab-project at the same time")
		os.Exit(1)
	}
	if !repoSpecifiedInline && projectsCsvPath == "" {
		logger.Error("must specify either -projects-csv or both of -github-repo and -gitlab-project")
		os.Exit(1)
	}

	if renameMasterToMain && renameTrunkBranch != "" {
		logger.Error("cannot specify -rename-master-to-main and -rename-trunk-branch together")
		os.Exit(1)
	}

	retryClient := &retryablehttp.Client{
		HTTPClient:   cleanhttp.DefaultPooledClient(),
		Logger:       nil,
		RetryMax:     100,
		RetryWaitMin: 1 * time.Second,
		RetryWaitMax: 10 * time.Second,
	}

	retryClient.Backoff = func(min, max time.Duration, attemptNum int, resp *http.Response) (sleep time.Duration) {
		requestMethod := "unknown"
		requestUrl := "unknown"
		statusCode := 0

		if resp != nil {
			statusCode = resp.StatusCode
			if req := resp.Request; req != nil {
				requestMethod = req.Method
				if req.URL != nil {
					requestUrl = req.URL.String()
				}
			}
		}

		defer func() {
			logger.Trace("waiting before retrying failed API request", "method", requestMethod, "url", requestUrl, "status", statusCode, "sleep", sleep, "attempt", attemptNum, "max_attempts", retryClient.RetryMax)
		}()

		if resp != nil {
			// Check the Retry-After header first (highest priority)
			if s, ok := resp.Header["Retry-After"]; ok && len(s) > 0 {
				if retryAfter, err := strconv.ParseInt(s[0], 10, 64); err == nil {
					sleep = time.Second * time.Duration(retryAfter)
					logger.Warn("waiting before retrying failed API request due to Retry-After header", "method", requestMethod, "url", requestUrl, "status", statusCode, "sleep", sleep, "attempt", attemptNum, "max_attempts", retryClient.RetryMax)
					return
				}
			}

			// Reference:
			// - https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28
			// - https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api?apiVersion=2022-11-28
			// Always check X-Ratelimit-Reset if present, regardless of remaining count.
			// This allows us to exit exponential backoff when we get a response with rate limit info.
			if w, ok := resp.Header["X-Ratelimit-Reset"]; ok && len(w) > 0 {
				if recoveryEpoch, err := strconv.ParseInt(w[0], 10, 64); err == nil {
					// Add 10 seconds to recovery timestamp for clock differences
					calculatedSleep := roundDuration(time.Until(time.Unix(recoveryEpoch+10, 0)), time.Second)
					// Ensure we don't sleep for negative durations (clock skew or already passed)
					if calculatedSleep > 0 {
						sleep = calculatedSleep
						logger.Warn("waiting before retrying failed API request due to X-Ratelimit-Reset header", "method", requestMethod, "url", requestUrl, "status", statusCode, "sleep", sleep, "attempt", attemptNum, "max_attempts", retryClient.RetryMax)
						return
					}
				}
			}

			// Check X-Ratelimit-Remaining for additional context
			if v, ok := resp.Header["X-Ratelimit-Remaining"]; ok && len(v) > 0 {
				if remaining, err := strconv.ParseInt(v[0], 10, 64); err == nil && remaining == 0 {
					// If we get here, X-Ratelimit-Reset was missing/invalid but remaining == 0
					// Fallback: wait for the minimum sleep duration
					sleep = min
					logger.Warn("waiting before retrying failed API request due to X-Ratelimit-Remaining header (fallback - X-Ratelimit-Reset missing)", "method", requestMethod, "url", requestUrl, "status", statusCode, "sleep", sleep, "attempt", attemptNum, "max_attempts", retryClient.RetryMax)
					return
				}
			}
		}

		// Exponential backoff
		mult := math.Pow(2, float64(attemptNum)) * float64(min)
		wait := time.Duration(mult)
		if float64(wait) != mult || wait > max {
			wait = max
		}

		sleep = wait
		logger.Warn("waiting before retrying failed API request due to exponential backoff", "method", requestMethod, "url", requestUrl, "status", statusCode, "sleep", sleep, "attempt", attemptNum, "max_attempts", retryClient.RetryMax)
		return
	}

	retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
		// Network errors should be retried
		if err != nil {
			return true, nil
		}

		// Potential connection reset
		if resp == nil {
			return true, nil
		}

		errResp := GitHubError{}
		// Unmarshal error response for better logging, but only for error status codes
		if resp.StatusCode >= 400 {
			if err = unmarshalResp(resp, &errResp); err != nil {
				// If we can't unmarshal, still proceed with retry logic
				// but log the unmarshal error
				logger.Trace("failed to unmarshal error response", "error", err)
			}
		}

		// Token not authorized for org - don't retry SAML enforcement errors
		if resp.StatusCode == http.StatusForbidden {
			if match, err := regexp.MatchString("SAML enforcement", errResp.Message); err != nil {
				return false, fmt.Errorf("matching 403 response: %v", err)
			} else if match {
				msg := errResp.Message
				if errResp.DocumentationURL != "" {
					msg += fmt.Sprintf(" - %s", errResp.DocumentationURL)
				}
				return false, fmt.Errorf("received 403 with response: %v", msg)
			}
		}

		retryableStatuses := []int{
			http.StatusTooManyRequests, // rate-limiting
			http.StatusForbidden,       // rate-limiting (non-SAML 403s)

			http.StatusRequestTimeout,
			http.StatusFailedDependency,
			http.StatusInternalServerError,
			http.StatusBadGateway,
			http.StatusServiceUnavailable,
			http.StatusGatewayTimeout,
		}

		requestMethod := "unknown"
		requestUrl := "unknown"

		if req := resp.Request; req != nil {
			requestMethod = req.Method
			if req.URL != nil {
				requestUrl = req.URL.String()
			}
		}

		for _, status := range retryableStatuses {
			if resp.StatusCode == status {
				logger.Trace("retrying failed API request", "method", requestMethod, "url", requestUrl, "status", resp.StatusCode, "message", errResp.Message)
				return true, nil
			}
		}

		return false, nil
	}

	transport := &gitHubAdvancedSearchModder{
		base: &retryablehttp.RoundTripper{Client: retryClient},
	}
	client := githubpagination.NewClient(transport, githubpagination.WithPerPage(100))

	if githubDomain == defaultGithubDomain {
		gh = github.NewClient(client).WithAuthToken(githubToken)
	} else {
		githubUrl := fmt.Sprintf("https://%s", githubDomain)
		if gh, err = github.NewClient(client).WithAuthToken(githubToken).WithEnterpriseURLs(githubUrl, githubUrl); err != nil {
			sendErr(err)
			os.Exit(1)
		}
	}

	gitlabOpts := make([]gitlab.ClientOptionFunc, 0)
	if gitlabDomain != defaultGitlabDomain {
		gitlabUrl := fmt.Sprintf("https://%s", gitlabDomain)
		gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(gitlabUrl))
	}
	if gl, err = gitlab.NewClient(gitlabToken, gitlabOpts...); err != nil {
		sendErr(err)
		os.Exit(1)
	}

	projects := make([]Project, 0)
	if projectsCsvPath != "" {
		data, err := os.ReadFile(projectsCsvPath)
		if err != nil {
			sendErr(err)
			os.Exit(1)
		}

		// Trim a UTF-8 BOM, if present
		data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))

		if projects, err = csv.NewReader(bytes.NewBuffer(data)).ReadAll(); err != nil {
			sendErr(err)
			os.Exit(1)
		}
	} else {
		projects = []Project{{gitlabProject, githubRepo}}
	}

	if report {
		printReport(ctx, projects)
	} else {
		if err = performMigration(ctx, projects); err != nil {
			sendErr(err)
			os.Exit(1)
		} else if errCount > 0 {
			logger.Warn(fmt.Sprintf("encountered %d errors during migration, review log output for details", errCount))
			os.Exit(1)
		}
	}
}

func printReport(ctx context.Context, projects []Project) {
	logger.Debug("building report")

	results := make([]Report, 0)

	for _, proj := range projects {
		if err := ctx.Err(); err != nil {
			return
		}

		result, err := reportProject(ctx, proj)
		if err != nil {
			errCount++
			sendErr(err)
		}

		if result != nil {
			results = append(results, *result)
		}
	}

	fmt.Println()

	totalMergeRequests := 0
	for _, result := range results {
		totalMergeRequests += result.MergeRequestsCount
		fmt.Printf("%#v\n", result)
	}

	fmt.Println()
	fmt.Printf("Total merge requests: %d\n", totalMergeRequests)
	fmt.Println()
}

func reportProject(_ context.Context, slugs []string) (*Report, error) {
	gitlabPath, _, err := parseProjectSlugs(slugs)
	if err != nil {
		return nil, fmt.Errorf("parsing project slugs: %v", err)
	}

	logger.Debug("searching for GitLab project", "name", gitlabPath[1], "group", gitlabPath[0])
	searchTerm := gitlabPath[1]
	projectResult, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: &searchTerm})
	if err != nil {
		return nil, fmt.Errorf("listing projects: %v", err)
	}

	var proj *gitlab.Project
	for _, item := range projectResult {
		if item == nil {
			continue
		}

		if item.PathWithNamespace == slugs[0] {
			logger.Debug("found GitLab project", "name", gitlabPath[1], "group", gitlabPath[0], "project_id", item.ID)
			proj = item
		}
	}

	if proj == nil {
		return nil, fmt.Errorf("no matching GitLab project found: %s", slugs[0])
	}

	var mergeRequests []*gitlab.MergeRequest

	opts := &gitlab.ListProjectMergeRequestsOptions{
		OrderBy: pointer("created_at"),
		Sort:    pointer("asc"),
	}

	logger.Debug("retrieving GitLab merge requests", "name", gitlabPath[1], "group", gitlabPath[0], "project_id", proj.ID)
	for {
		result, resp, err := gl.MergeRequests.ListProjectMergeRequests(proj.ID, opts)
		if err != nil {
			return nil, fmt.Errorf("retrieving gitlab merge requests: %v", err)
		}

		mergeRequests = append(mergeRequests, result...)

		if resp.NextPage == 0 {
			break
		}

		opts.Page = resp.NextPage
	}

	return &Report{
		GroupName:          gitlabPath[0],
		ProjectName:        gitlabPath[1],
		MergeRequestsCount: len(mergeRequests),
	}, nil
}

func performMigration(ctx context.Context, projects []Project) error {
	concurrency := maxConcurrency
	if len(projects) < maxConcurrency {
		concurrency = len(projects)
	}

	logger.Info(fmt.Sprintf("processing %d project(s) with %d workers", len(projects), concurrency))

	var wg sync.WaitGroup
	queue := make(chan Project, concurrency*2)

	for i := 0; i < concurrency; i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()

			for slugs := range queue {
				if err := ctx.Err(); err != nil {
					break
				}

				proj, err := newProject(slugs)
				if err != nil {
					errCount++
					sendErr(err)
					continue
				}

				if err := proj.migrate(ctx); err != nil {
					errCount++
					sendErr(err)
				}
			}
		}()
	}

	queueProjects := func() {
		for _, proj := range projects {
			if err := ctx.Err(); err != nil {
				break
			}

			queue <- proj
		}
	}

	if loop {
		logger.Info(fmt.Sprintf("looping migration until canceled"))
		for {
			if err := ctx.Err(); err != nil {
				break
			}

			queueProjects()
		}
	} else {
		queueProjects()
		close(queue)
	}

	wg.Wait()

	return nil
}