Skip to content

Commit 418b92a

Browse files
committed
Phase 7: Port BBS archive downloaders (SSH + SMB)
Add SSH (SFTP) and SMB archive download capabilities to the BBS package. These downloaders are used by bbs2gh migrate-repo to download Bitbucket Server export archives before uploading them for GitHub import. SSH downloader: - Uses golang.org/x/crypto/ssh + github.com/pkg/sftp - sftpClient interface (Stat/Open/Close) for testability - Progress logging with rate-limited output - UserError with --bbs-shared-home hint when using default Linux path SMB downloader: - Uses github.com/hirochachacha/go-smb2 - smbConnector + smbShare interfaces for testability - Connect → Login → Mount → Read loop with cleanup - UserError with --bbs-shared-home hint when using default Windows path Shared infrastructure (archive.go): - Constants: ExportArchiveSourceDirectory, DefaultTargetDirectory, DefaultBbsSharedHomeDirectory{Linux,Windows} - Path helpers: ExportArchiveFileName, SourceExportArchiveAbsolutePath - fileSystem interface + osFileSystem for testable file operations - progressLogger with mutex-protected rate limiting - copyWithProgress shared copy loop - logFriendlySize and percentage formatters New files: archive.go, ssh_downloader.go, sftp_client.go, ssh_downloader_test.go, smb_downloader.go, smb_client.go, smb_downloader_test.go 15 new tests (6 SSH + 9 SMB), all passing. 0 lint issues.
1 parent b9bd952 commit 418b92a

12 files changed

Lines changed: 1997 additions & 217 deletions

go.mod

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@ require (
1010
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3
1111
github.com/google/go-github/v68 v68.0.0
1212
github.com/google/uuid v1.6.0
13+
github.com/hirochachacha/go-smb2 v1.1.0
14+
github.com/pkg/sftp v1.13.10
1315
github.com/spf13/cobra v1.10.2
1416
github.com/stretchr/testify v1.11.1
17+
golang.org/x/crypto v0.45.0
1518
)
1619

1720
require (
@@ -27,11 +30,14 @@ require (
2730
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
2831
github.com/aws/smithy-go v1.24.2 // indirect
2932
github.com/davecgh/go-spew v1.1.1 // indirect
33+
github.com/geoffgarside/ber v1.1.0 // indirect
3034
github.com/google/go-querystring v1.1.0 // indirect
3135
github.com/inconshreveable/mousetrap v1.1.0 // indirect
36+
github.com/kr/fs v0.1.0 // indirect
3237
github.com/pmezard/go-difflib v1.0.0 // indirect
3338
github.com/spf13/pflag v1.0.9 // indirect
3439
golang.org/x/net v0.47.0 // indirect
40+
golang.org/x/sys v0.38.0 // indirect
3541
golang.org/x/text v0.31.0 // indirect
3642
gopkg.in/yaml.v3 v3.0.1 // indirect
3743
)

go.sum

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqx
3939
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
4040
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
4141
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
42+
github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
43+
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
4244
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
4345
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
4446
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -50,8 +52,12 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD
5052
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
5153
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
5254
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
55+
github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
56+
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
5357
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
5458
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
59+
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
60+
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
5561
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
5662
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
5763
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -60,6 +66,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
6066
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
6167
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
6268
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
69+
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
70+
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
6371
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
6472
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
6573
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -72,12 +80,20 @@ github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
7280
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
7381
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
7482
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
83+
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
84+
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
7585
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
7686
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
87+
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
7788
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
7889
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
90+
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
91+
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
7992
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
8093
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
94+
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
95+
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
96+
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
8197
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
8298
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
8399
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/bbs/archive.go

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package bbs
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"os"
7+
"path/filepath"
8+
"sync"
9+
"time"
10+
11+
"github.com/github/gh-gei/pkg/logger"
12+
)
13+
14+
// Archive download constants matching C# IBbsArchiveDownloader and BbsSettings.
15+
const (
16+
ExportArchiveSourceDirectory = "data/migration/export"
17+
DefaultTargetDirectory = "bbs_archive_downloads"
18+
DefaultBbsSharedHomeDirectoryLinux = "/var/atlassian/application-data/bitbucket/shared"
19+
DefaultBbsSharedHomeDirectoryWindows = `c$\atlassian\applicationdata\bitbucket\shared`
20+
downloadProgressReportInterval = 10 * time.Second
21+
)
22+
23+
// ExportArchiveFileName returns the archive filename for an export job.
24+
func ExportArchiveFileName(exportJobID int64) string {
25+
return fmt.Sprintf("Bitbucket_export_%d.tar", exportJobID)
26+
}
27+
28+
// SourceExportArchiveAbsolutePath returns the full path to the export archive on the BBS server.
29+
func SourceExportArchiveAbsolutePath(bbsSharedHome string, exportJobID int64) string {
30+
return filepath.ToSlash(filepath.Join(bbsSharedHome, ExportArchiveSourceDirectory, ExportArchiveFileName(exportJobID)))
31+
}
32+
33+
// fileSystem abstracts filesystem operations for testability.
34+
type fileSystem interface {
35+
MkdirAll(path string, perm os.FileMode) error
36+
Create(path string) (io.WriteCloser, error)
37+
}
38+
39+
// osFileSystem is the default fileSystem implementation using the real OS.
40+
type osFileSystem struct{}
41+
42+
func (osFileSystem) MkdirAll(path string, perm os.FileMode) error {
43+
return os.MkdirAll(path, perm)
44+
}
45+
46+
func (osFileSystem) Create(path string) (io.WriteCloser, error) {
47+
return os.Create(path)
48+
}
49+
50+
// progressLogger tracks download progress with rate-limited log output.
51+
type progressLogger struct {
52+
log *logger.Logger
53+
mu sync.Mutex
54+
nextProgressTime time.Time
55+
}
56+
57+
func newProgressLogger(log *logger.Logger) *progressLogger {
58+
return &progressLogger{
59+
log: log,
60+
nextProgressTime: time.Now(),
61+
}
62+
}
63+
64+
func (p *progressLogger) logProgress(downloadedBytes, totalBytes int64) {
65+
p.mu.Lock()
66+
defer p.mu.Unlock()
67+
68+
if time.Now().Before(p.nextProgressTime) {
69+
return
70+
}
71+
72+
if totalBytes > 0 {
73+
p.log.Info(
74+
"Archive download in progress, %s out of %s (%s) completed...",
75+
logFriendlySize(downloadedBytes),
76+
logFriendlySize(totalBytes),
77+
percentage(downloadedBytes, totalBytes),
78+
)
79+
} else {
80+
p.log.Info("Archive download in progress, %s completed...", logFriendlySize(downloadedBytes))
81+
}
82+
83+
p.nextProgressTime = p.nextProgressTime.Add(downloadProgressReportInterval)
84+
}
85+
86+
func percentage(downloaded, total int64) string {
87+
if total == 0 {
88+
return "unknown%"
89+
}
90+
pct := int(float64(downloaded) * 100.0 / float64(total))
91+
return fmt.Sprintf("%d%%", pct)
92+
}
93+
94+
func logFriendlySize(size int64) string {
95+
const (
96+
kilobyte = 1024
97+
megabyte = 1024 * kilobyte
98+
gigabyte = 1024 * megabyte
99+
)
100+
101+
switch {
102+
case size < kilobyte:
103+
return fmt.Sprintf("%d bytes", size)
104+
case size < megabyte:
105+
return fmt.Sprintf("%.0f KB", float64(size)/float64(kilobyte))
106+
case size < gigabyte:
107+
return fmt.Sprintf("%.0f MB", float64(size)/float64(megabyte))
108+
default:
109+
return fmt.Sprintf("%.2f GB", float64(size)/float64(gigabyte))
110+
}
111+
}
112+
113+
// copyWithProgress copies from src to dst, reporting download progress.
114+
func copyWithProgress(src io.Reader, dst io.Writer, totalSize int64, log *logger.Logger) error {
115+
progress := newProgressLogger(log)
116+
buf := make([]byte, 64*1024)
117+
var downloaded int64
118+
for {
119+
n, readErr := src.Read(buf)
120+
if n > 0 {
121+
if _, writeErr := dst.Write(buf[:n]); writeErr != nil {
122+
return fmt.Errorf("write to local file: %w", writeErr)
123+
}
124+
downloaded += int64(n)
125+
progress.logProgress(downloaded, totalSize)
126+
}
127+
if readErr == io.EOF {
128+
break
129+
}
130+
if readErr != nil {
131+
return fmt.Errorf("read from remote: %w", readErr)
132+
}
133+
}
134+
return nil
135+
}

0 commit comments

Comments
 (0)