Skip to content

Commit fcfdb5d

Browse files
offbyoneoffbyone
committed
Phase 9: Wire live constructors, fix auth/parsing/validation bugs for CI
- Wire ado2gh generate-script live constructor with real ADO client + inspector - Wire gei migrate-repo storage backends (Azure, AWS, GitHub-owned) - Wire bbs2gh migrate-repo GitHub-owned storage - Fix tokenRoundTripper auth: use "Bearer" instead of "token" (gei + bbs2gh) - Fix ado2gh + bbs2gh main.go: print errors before os.Exit(1) instead of silencing - Add Content-Type: application/octet-stream header to ghowned single upload - Include response body in ghowned client error messages for better debugging - Fix golangci-lint v2 issues (5 fixes across alerts and github packages) - Fix ADO repo model: remove ,string JSON tags from Size/IsDisabled (ADO API returns numbers/booleans, not strings) - Fix BBS hasAWSSubOptions: only check CLI flags, not env vars (matches C# behavior; env vars like AWS_ACCESS_KEY_ID should not trigger bucket-name validation) - Fix ghowned parseURIResponse: use map[string]interface{} to handle mixed-type API responses (size is a number, not a string)
1 parent 5514b43 commit fcfdb5d

20 files changed

Lines changed: 508 additions & 124 deletions

‎.github/workflows/go-ci.yml‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,18 +49,19 @@ jobs:
4949
- name: Build Go binaries
5050
run: just go-build
5151

52-
- name: Create coverage directory
53-
run: mkdir -p coverage
54-
shell: bash
55-
5652
- name: Run Go tests
57-
run: go test -v -race -coverprofile=./coverage/coverage.out ./...
53+
run: go test -v -race ./...
54+
if: matrix.runner-os != 'ubuntu-latest'
55+
56+
- name: Run Go tests with coverage
57+
run: go test -v -race -coverprofile=coverage.out ./...
58+
if: matrix.runner-os == 'ubuntu-latest'
5859

5960
- name: Generate coverage report
6061
if: matrix.runner-os == 'ubuntu-latest'
6162
run: |
62-
go tool cover -html=./coverage/coverage.out -o coverage.html
63-
go tool cover -func=./coverage/coverage.out
63+
go tool cover -html=coverage.out -o coverage.html
64+
go tool cover -func=coverage.out
6465
6566
- name: Upload coverage artifact
6667
if: matrix.runner-os == 'ubuntu-latest'

‎cmd/ado2gh/generate_script.go‎

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strings"
88

99
"github.com/github/gh-gei/pkg/ado"
10+
"github.com/github/gh-gei/pkg/env"
1011
"github.com/github/gh-gei/pkg/logger"
1112
"github.com/github/gh-gei/pkg/scriptgen"
1213
"github.com/spf13/cobra"
@@ -43,6 +44,7 @@ type generateScriptArgs struct {
4344
githubOrg string
4445
adoOrg string
4546
adoTeamProject string
47+
adoPAT string
4648
output string
4749
sequential bool
4850
adoServerURL string
@@ -132,11 +134,60 @@ func newGenerateScriptCmd(
132134
// ---------------------------------------------------------------------------
133135

134136
func newGenerateScriptCmdLive() *cobra.Command {
135-
// TODO: wire up real ADO client and inspector
136-
return &cobra.Command{
137+
var a generateScriptArgs
138+
139+
cmd := &cobra.Command{
137140
Use: "generate-script",
138141
Short: "Generates a migration script",
142+
Long: "Generates a PowerShell script that automates an Azure DevOps to GitHub migration.",
143+
RunE: func(cmd *cobra.Command, _ []string) error {
144+
log := getLogger(cmd)
145+
envProv := env.New()
146+
147+
adoPAT := a.adoPAT
148+
if adoPAT == "" {
149+
adoPAT = envProv.ADOPAT()
150+
}
151+
152+
adoServerURL := a.adoServerURL
153+
if adoServerURL == "" {
154+
adoServerURL = "https://dev.azure.com"
155+
}
156+
157+
client := ado.NewClient(adoServerURL, adoPAT, log)
158+
ins := ado.NewInspector(log, client)
159+
ins.OrgFilter = a.adoOrg
160+
ins.TeamProjectFilter = a.adoTeamProject
161+
162+
return runGenerateScript(cmd.Context(), client, ins, log, a, defaultWriteToFile)
163+
},
139164
}
165+
166+
// Required flags
167+
cmd.Flags().StringVar(&a.githubOrg, "github-org", "", "Target GitHub organization name (REQUIRED)")
168+
169+
// Optional flags
170+
cmd.Flags().StringVar(&a.adoOrg, "ado-org", "", "Azure DevOps organization name")
171+
cmd.Flags().StringVar(&a.adoTeamProject, "ado-team-project", "", "Azure DevOps team project name")
172+
cmd.Flags().StringVar(&a.output, "output", "./migrate.ps1", "Output file path for the migration script")
173+
cmd.Flags().BoolVar(&a.sequential, "sequential", false, "Generate a sequential (non-parallel) script")
174+
cmd.Flags().StringVar(&a.adoServerURL, "ado-server-url", "", "Azure DevOps Server URL")
175+
cmd.Flags().StringVar(&a.targetAPIURL, "target-api-url", "", "API URL for the target GitHub instance")
176+
cmd.Flags().BoolVar(&a.createTeams, "create-teams", false, "Include team creation and assignment scripts")
177+
cmd.Flags().BoolVar(&a.linkIdpGroups, "link-idp-groups", false, "Link IdP groups to teams")
178+
cmd.Flags().BoolVar(&a.lockAdoRepos, "lock-ado-repos", false, "Lock ADO repos before migration")
179+
cmd.Flags().BoolVar(&a.disableAdoRepos, "disable-ado-repos", false, "Disable ADO repos after migration")
180+
cmd.Flags().BoolVar(&a.rewirePipelines, "rewire-pipelines", false, "Rewire Azure Pipelines to GitHub repos")
181+
cmd.Flags().BoolVar(&a.downloadMigrationLogs, "download-migration-logs", false, "Download migration logs after migration")
182+
cmd.Flags().BoolVar(&a.all, "all", false, "Enable all optional migration steps")
183+
cmd.Flags().StringVar(&a.repoList, "repo-list", "", "Path to a CSV file with repos to migrate")
184+
cmd.Flags().StringVar(&a.adoPAT, "ado-pat", "", "")
185+
186+
// Hidden flags
187+
_ = cmd.Flags().MarkHidden("ado-server-url")
188+
_ = cmd.Flags().MarkHidden("ado-pat")
189+
190+
return cmd
140191
}
141192

142193
// ---------------------------------------------------------------------------
@@ -749,6 +800,6 @@ func wrap(script string) string {
749800
}
750801

751802
// defaultWriteToFile writes content to a file (production implementation).
752-
func defaultWriteToFile(path, content string) error { //nolint:unused // will be used when newGenerateScriptCmdLive is fully wired
803+
func defaultWriteToFile(path, content string) error {
753804
return os.WriteFile(path, []byte(content), 0o600)
754805
}

‎cmd/ado2gh/main.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@ package main
22

33
import (
44
"context"
5+
"errors"
6+
"fmt"
57
"net/http"
68
"os"
79
"strings"
810

11+
"github.com/github/gh-gei/internal/cmdutil"
912
"github.com/github/gh-gei/pkg/env"
1013
"github.com/github/gh-gei/pkg/logger"
1114
"github.com/github/gh-gei/pkg/status"
@@ -24,7 +27,18 @@ var (
2427
)
2528

2629
func main() {
27-
if err := newRootCmd().Execute(); err != nil {
30+
rootCmd := newRootCmd()
31+
if err := rootCmd.Execute(); err != nil {
32+
if log, ok := rootCmd.Context().Value(loggerKey).(*logger.Logger); ok && log != nil {
33+
var userErr *cmdutil.UserError
34+
if errors.As(err, &userErr) {
35+
log.Errorf("%v", err)
36+
} else {
37+
log.Errorf("Unexpected error: %v", err)
38+
}
39+
} else {
40+
fmt.Fprintf(os.Stderr, "[ERROR] %v\n", err)
41+
}
2842
os.Exit(1)
2943
}
3044
}

‎cmd/ado2gh/wiring.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,8 +163,8 @@ func newDownloadLogsCmdLive() *cobra.Command {
163163
}
164164

165165
cmd.Flags().StringVar(&migrationID, "migration-id", "", "The ID of the migration")
166-
cmd.Flags().StringVar(&githubTargetOrg, "github-target-org", "", "Target GitHub organization")
167-
cmd.Flags().StringVar(&targetRepo, "target-repo", "", "Target repository name")
166+
cmd.Flags().StringVar(&githubTargetOrg, "github-org", "", "Target GitHub organization")
167+
cmd.Flags().StringVar(&targetRepo, "github-repo", "", "Target repository name")
168168
cmd.Flags().StringVar(&logFile, "migration-log-file", "", "Custom output filename for the migration log")
169169
cmd.Flags().BoolVar(&overwrite, "overwrite", false, "Overwrite the log file if it already exists")
170170
cmd.Flags().StringVar(&githubTargetPAT, "github-target-pat", "", "Personal access token for the target GitHub instance")

‎cmd/bbs2gh/main.go‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@ package main
22

33
import (
44
"context"
5+
"errors"
6+
"fmt"
57
"net/http"
68
"os"
79
"strings"
810

11+
"github.com/github/gh-gei/internal/cmdutil"
912
"github.com/github/gh-gei/pkg/env"
1013
"github.com/github/gh-gei/pkg/logger"
1114
"github.com/github/gh-gei/pkg/status"
@@ -24,7 +27,18 @@ var (
2427
)
2528

2629
func main() {
27-
if err := newRootCmd().Execute(); err != nil {
30+
rootCmd := newRootCmd()
31+
if err := rootCmd.Execute(); err != nil {
32+
if log, ok := rootCmd.Context().Value(loggerKey).(*logger.Logger); ok && log != nil {
33+
var userErr *cmdutil.UserError
34+
if errors.As(err, &userErr) {
35+
log.Errorf("%v", err)
36+
} else {
37+
log.Errorf("Unexpected error: %v", err)
38+
}
39+
} else {
40+
fmt.Fprintf(os.Stderr, "[ERROR] %v\n", err)
41+
}
2842
os.Exit(1)
2943
}
3044
}

‎cmd/bbs2gh/migrate_repo.go‎

Lines changed: 64 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ import (
44
"context"
55
"fmt"
66
"io"
7+
"net/http"
78
"net/url"
9+
"strconv"
810
"strings"
911
"time"
1012

@@ -19,6 +21,7 @@ import (
1921
"github.com/github/gh-gei/pkg/migration"
2022
awsStorage "github.com/github/gh-gei/pkg/storage/aws"
2123
azureStorage "github.com/github/gh-gei/pkg/storage/azure"
24+
"github.com/github/gh-gei/pkg/storage/ghowned"
2225
"github.com/google/uuid"
2326
"github.com/spf13/cobra"
2427
)
@@ -127,6 +130,7 @@ type bbsMigrateRepoArgs struct {
127130
awsRegion string
128131
keepArchive bool
129132
targetAPIURL string
133+
targetUploadsURL string
130134
queueOnly bool
131135
useGithubStorage bool
132136
}
@@ -393,7 +397,7 @@ func validateBbsUploadOptions(a *bbsMigrateRepoArgs, envProv bbsMigrateRepoEnvPr
393397
shouldUseAzure := resolveBbsAzureConnectionString(a.azureStorageConnectionString, envProv) != ""
394398
shouldUseAWS := a.awsBucketName != ""
395399

396-
if err := validateBbsUploadConflicts(a, shouldUseAzure, shouldUseAWS, envProv); err != nil {
400+
if err := validateBbsUploadConflicts(a, shouldUseAzure, shouldUseAWS); err != nil {
397401
return err
398402
}
399403

@@ -404,8 +408,8 @@ func validateBbsUploadOptions(a *bbsMigrateRepoArgs, envProv bbsMigrateRepoEnvPr
404408
return nil
405409
}
406410

407-
func validateBbsUploadConflicts(a *bbsMigrateRepoArgs, shouldUseAzure, shouldUseAWS bool, envProv bbsMigrateRepoEnvProvider) error {
408-
if !shouldUseAWS && hasAWSSubOptions(a, envProv) {
411+
func validateBbsUploadConflicts(a *bbsMigrateRepoArgs, shouldUseAzure, shouldUseAWS bool) error {
412+
if !shouldUseAWS && hasAWSSubOptions(a) {
409413
return cmdutil.NewUserError("The AWS S3 bucket name must be provided with --aws-bucket-name if other AWS S3 upload options are set.")
410414
}
411415
if a.useGithubStorage && shouldUseAWS {
@@ -427,11 +431,11 @@ func validateBbsUploadConflicts(a *bbsMigrateRepoArgs, shouldUseAzure, shouldUse
427431
return nil
428432
}
429433

430-
func hasAWSSubOptions(a *bbsMigrateRepoArgs, envProv bbsMigrateRepoEnvProvider) bool {
431-
return a.awsAccessKey != "" || resolveBbsAWSAccessKey("", envProv) != "" ||
432-
a.awsSecretKey != "" || resolveBbsAWSSecretKey("", envProv) != "" ||
433-
a.awsSessionToken != "" || envProv.AWSSessionToken() != "" ||
434-
a.awsRegion != "" || resolveBbsAWSRegion("", envProv) != ""
434+
func hasAWSSubOptions(a *bbsMigrateRepoArgs) bool {
435+
return a.awsAccessKey != "" ||
436+
a.awsSecretKey != "" ||
437+
a.awsSessionToken != "" ||
438+
a.awsRegion != ""
435439
}
436440

437441
func validateBbsAWSCredentials(a *bbsMigrateRepoArgs, envProv bbsMigrateRepoEnvProvider) error {
@@ -689,8 +693,7 @@ func pollBbsExport(ctx context.Context, bbsAPI bbsMigrateRepoBbsAPI, exportID in
689693
return nil
690694
}
691695

692-
if upper != "INITIALISING" && upper != "IN_PROGRESS" { //nolint:misspell // BBS API uses British spelling
693-
// Error state
696+
if upper == "FAILED" || upper == "ABORTED" {
694697
return cmdutil.NewUserErrorf("BBS export failed with state: %s - %s", exportState, message)
695698
}
696699

@@ -788,6 +791,17 @@ type awsLogAdapter struct {
788791

789792
func (a *awsLogAdapter) LogInfo(format string, args ...interface{}) { a.log.Info(format, args...) }
790793

794+
// bbsTokenRoundTripper attaches a Bearer token to every outgoing request.
795+
type bbsTokenRoundTripper struct {
796+
token string
797+
}
798+
799+
func (t *bbsTokenRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
800+
req = req.Clone(req.Context())
801+
req.Header.Set("Authorization", "Bearer "+t.token)
802+
return http.DefaultTransport.RoundTrip(req)
803+
}
804+
791805
// ---------------------------------------------------------------------------
792806
// Production command constructor (used by main.go)
793807
// ---------------------------------------------------------------------------
@@ -859,6 +873,7 @@ func newMigrateRepoCmdLive() *cobra.Command {
859873
cmd.Flags().StringVar(&a.githubPAT, "github-pat", "", "Personal access token for the target GitHub instance")
860874
cmd.Flags().StringVar(&a.targetRepoVisibility, "target-repo-visibility", "", "Target repository visibility (public, private, internal)")
861875
cmd.Flags().StringVar(&a.targetAPIURL, "target-api-url", bbsDefaultTargetAPIURL, "API URL for the target GitHub instance")
876+
cmd.Flags().StringVar(&a.targetUploadsURL, "target-uploads-url", "", "Uploads URL for the target GitHub instance")
862877

863878
// Upload storage flags
864879
cmd.Flags().StringVar(&a.azureStorageConnectionString, "azure-storage-connection-string", "", "Azure Blob Storage connection string")
@@ -873,6 +888,9 @@ func newMigrateRepoCmdLive() *cobra.Command {
873888
cmd.Flags().BoolVar(&a.queueOnly, "queue-only", false, "Queue the migration without waiting for completion")
874889
cmd.Flags().BoolVar(&a.useGithubStorage, "use-github-storage", false, "Use GitHub-owned storage for archives")
875890

891+
// Hidden flags
892+
_ = cmd.Flags().MarkHidden("target-uploads-url")
893+
876894
return cmd
877895
}
878896

@@ -970,7 +988,42 @@ func buildBbsArchiveUploader(a *bbsMigrateRepoArgs, envProv bbsMigrateRepoEnvPro
970988
}
971989

972990
if a.useGithubStorage {
973-
log.Warning("GitHub-owned storage is not yet fully implemented in the Go port")
991+
uploadsURL := a.targetUploadsURL
992+
if uploadsURL == "" {
993+
uploadsURL = "https://uploads.github.com"
994+
}
995+
996+
// Resolve target token for the ghowned HTTP client
997+
targetToken := resolveBbsTargetToken(a.githubPAT, envProv)
998+
999+
ghHTTPClient := &http.Client{
1000+
Transport: &bbsTokenRoundTripper{token: targetToken},
1001+
}
1002+
1003+
var ghOwnedOpts []ghowned.Option
1004+
ghOwnedOpts = append(ghOwnedOpts, ghowned.WithLogger(log))
1005+
1006+
envReal := env.New()
1007+
if mebiStr := envReal.GitHubOwnedStorageMultipartMebibytes(); mebiStr != "" {
1008+
if mebi, err := strconv.ParseInt(mebiStr, 10, 64); err == nil {
1009+
ghOwnedOpts = append(ghOwnedOpts, ghowned.WithPartSizeMebibytes(mebi))
1010+
}
1011+
}
1012+
1013+
ghOwnedClient := ghowned.NewClient(uploadsURL, ghHTTPClient, ghOwnedOpts...)
1014+
1015+
// Build a GitHub client for org ID resolution
1016+
tgtAPI := a.targetAPIURL
1017+
if tgtAPI == "" {
1018+
tgtAPI = bbsDefaultTargetAPIURL
1019+
}
1020+
targetGH := github.NewClient(targetToken,
1021+
github.WithAPIURL(tgtAPI),
1022+
github.WithLogger(log),
1023+
github.WithVersion(version),
1024+
)
1025+
1026+
uploaderOpts = append(uploaderOpts, archive.WithGitHub(ghOwnedClient, targetGH))
9741027
}
9751028

9761029
return archive.NewUploader(uploaderOpts...), nil

0 commit comments

Comments
 (0)