diff --git a/go.mod b/go.mod index 0d0c721..e99838b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.26.0 toolchain go1.26.7 require ( + github.com/git-pkgs/history v0.1.1 github.com/git-pkgs/licenses v0.8.0 + github.com/git-pkgs/roles v0.0.0-20260917090927-eb44e8aa553d github.com/spf13/cobra v1.10.2 ) @@ -34,10 +36,10 @@ require ( github.com/git-pkgs/spdx v0.3.2 // indirect github.com/git-pkgs/vers v0.7.0 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect - github.com/go-git/go-billy/v6 v6.0.0-alpha.2 - github.com/go-git/go-git/v6 v6.0.0-alpha.4 + github.com/go-git/go-billy/v6 v6.0.0-alpha.2 // indirect + github.com/go-git/go-git/v6 v6.0.0-alpha.5 github.com/kevinburke/ssh_config v1.6.0 // indirect - github.com/klauspost/compress v1.20.0 + github.com/klauspost/compress v1.20.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pjbgf/sha1cd v0.6.0 // indirect @@ -47,7 +49,3 @@ require ( golang.org/x/sys v0.47.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -replace github.com/go-git/go-git/v6 => github.com/git-pkgs/go-git/v6 v6.0.0-alpha.4.0.20260906204421-52ea7303c392 - -replace github.com/go-git/go-billy/v6 => github.com/git-pkgs/go-billy/v6 v6.0.0-alpha.2.0.20260906202738-23c9d86c8d17 diff --git a/go.sum b/go.sum index eb43e96..42279c9 100644 --- a/go.sum +++ b/go.sum @@ -17,13 +17,12 @@ github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/git-pkgs/go-billy/v6 v6.0.0-alpha.2.0.20260906202738-23c9d86c8d17 h1:PzXJag1pssE3Hhl/tccdaPOtz5j2eZyl1WyUmai71FU= -github.com/git-pkgs/go-billy/v6 v6.0.0-alpha.2.0.20260906202738-23c9d86c8d17/go.mod h1:QGOWN4CzmtRfpFeCw5paG3RJyKdTwsfGASkxBVkIaAU= -github.com/git-pkgs/go-git/v6 v6.0.0-alpha.4.0.20260906204421-52ea7303c392 h1:V4/D19lZAHihN/jCHa5lIL6BobOjhTkWFtsOnq/bhYU= -github.com/git-pkgs/go-git/v6 v6.0.0-alpha.4.0.20260906204421-52ea7303c392/go.mod h1:2eK2w/x2zu4GWIHvrPUnxOTKkQ1H9Hm40hDPLl1az9Q= +github.com/git-pkgs/history v0.1.1 h1:WZZXMXSaZxU/JboXtiohfPvIv4RNZibPJuNw/BcWSFI= +github.com/git-pkgs/history v0.1.1/go.mod h1:wZtYwhJsU5LJFLhrw9Lvx4Ue3ESp7+tv4an9KEeVBns= github.com/git-pkgs/licenses v0.8.0 h1:nGeWm02M2ei6CXwsIkGwDIYqLBx3ocwwPDDG5TNLSSM= github.com/git-pkgs/licenses v0.8.0/go.mod h1:5GW02cjho+lLik/v2XdyVTTq3aNkMNe3nFAvF75mQTg= github.com/git-pkgs/magic v0.3.1 h1:UzjFRyEwJITA/JgznjmIM4VwuBszD2K4Q8XHgkgL+DM= @@ -34,6 +33,8 @@ github.com/git-pkgs/pom v0.1.7 h1:4yKdtw6eyShtjul6bcZdyz7yLQ+jdrYeYkKbskDGi4c= github.com/git-pkgs/pom v0.1.7/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= github.com/git-pkgs/purl v0.1.20 h1:a4qzvUy5mBZ2GGjOQNW2h/ocFqjTjOiTDMv2ONtivmM= github.com/git-pkgs/purl v0.1.20/go.mod h1:hthV5mp+Q67HpQ9+LnRLLmsReu5ooyQ5EaJsCGrA8yE= +github.com/git-pkgs/roles v0.0.0-20260917090927-eb44e8aa553d h1:BbX6xoEro5uIMhVY3l17hr/Zt7MYht8NhuEGSTLFK2Q= +github.com/git-pkgs/roles v0.0.0-20260917090927-eb44e8aa553d/go.mod h1:dRibEXgftewf7z8HKAfj6taRhqRQm/y0bdgQtk1fjg8= github.com/git-pkgs/spdx v0.3.2 h1:PV0w09Bt8rIkAof1+zuo3vxNYKgCoFTxQY9E1Q85PL0= github.com/git-pkgs/spdx v0.3.2/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.0 h1:7PD2DKFB8jTDIfiyWXbqYW54iVuNkFCBgXHgxOTdr8A= @@ -44,8 +45,12 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo= github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs= +github.com/go-git/go-billy/v6 v6.0.0-alpha.2 h1:1Sv5WemXL8CxKrAx1gioJ+uHNb2bZJhiQLfwSZ4Et8c= +github.com/go-git/go-billy/v6 v6.0.0-alpha.2/go.mod h1:r/bsv9i/iDyyEU8/Z6mjC+YraOVwie1ddfUqBCElKXQ= github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1 h1:gmqi2jvsreu0s8JMLylYDFq4sbjHwwlhktMw0DUg3mA= github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1/go.mod h1:ECf1MqJlBdYpKggBrOXjo/0EnvRZx6D++I86UYjPgAQ= +github.com/go-git/go-git/v6 v6.0.0-alpha.5 h1:sE+OlkHgYWNMVmN1s9sR7uyFgsWLtxcNWse/vBYKxRE= +github.com/go-git/go-git/v6 v6.0.0-alpha.5/go.mod h1:3IjhiZnM+uBmUrOGSeqrJpsmi4Vd0H2NZO/uK2a7d0s= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= @@ -67,6 +72,7 @@ github.com/package-url/packageurl-go v0.1.7/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -80,11 +86,9 @@ github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= -github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= diff --git a/gogit_history.go b/gogit_history.go index 7fc7b7e..1a0a60a 100644 --- a/gogit_history.go +++ b/gogit_history.go @@ -1,153 +1,45 @@ package main -import ( - "errors" - "fmt" - "io" - "os" - "strings" - - gogit "github.com/go-git/go-git/v6" - "github.com/go-git/go-git/v6/plumbing" - "github.com/go-git/go-git/v6/plumbing/object" - "github.com/go-git/go-git/v6/storage/filesystem" -) +import githistory "github.com/git-pkgs/history" func walkChangesGoGit(path string, merges bool, visit func(change)) error { - r, err := openGoGit(path) + r, err := openHistory(path) if err != nil { return err } defer func() { _ = r.Close() }() - return walkChangesGoGitStream(r, merges, visit) -} - -func historyRoots(r *gogit.Repository) ([]plumbing.Hash, error) { - refs, err := r.References() - if err != nil { - return nil, err - } - defer refs.Close() - var roots []plumbing.Hash - head, err := r.Head() - if err != nil && !errors.Is(err, plumbing.ErrReferenceNotFound) { - return nil, err - } - if err == nil { - h, ok, err := commitAtReference(r, head.Hash()) - if err != nil { - return nil, err - } - if ok { - roots = append(roots, h) - } - } - for { - ref, err := refs.Next() - if err == io.EOF { - return roots, nil - } - if err != nil { - return nil, err - } - if !strings.HasPrefix(ref.Name().String(), "refs/") { - continue - } - if strings.HasPrefix(ref.Name().String(), "refs/replace/") { - return nil, fmt.Errorf("gogit backend does not support replacement refs") - } - resolved, err := r.Reference(ref.Name(), true) - if err != nil { - return nil, err - } - h, ok, err := commitAtReference(r, resolved.Hash()) - if err != nil { - return nil, fmt.Errorf("reference %s: %w", ref.Name(), err) - } - if ok { - roots = append(roots, h) - } - } -} - -func commitAtReference(r *gogit.Repository, h plumbing.Hash) (plumbing.Hash, bool, error) { - seen := make(map[plumbing.Hash]bool) - for { - if seen[h] { - return plumbing.ZeroHash, false, fmt.Errorf("cyclic tag at %s", h) - } - seen[h] = true - o, err := r.Object(plumbing.AnyObject, h) - if err != nil { - return plumbing.ZeroHash, false, err - } - switch v := o.(type) { - case *object.Commit: - return v.Hash, true, nil - case *object.Tag: - h = v.Target - default: - return plumbing.ZeroHash, false, nil - } - } -} - -func commitSubject(message string) string { - var lines []string - for _, line := range strings.Split(strings.TrimSpace(message), "\n") { - if strings.TrimSpace(line) == "" { - break - } - lines = append(lines, strings.TrimSpace(line)) - } - return strings.Join(lines, " ") + return r.WalkChanges(githistory.ChangeOptions{ + Merges: merges, + Workers: *historyWorkers, + }, func(c githistory.Change) error { + visit(changeFromHistory(c)) + return nil + }) } func isShallowGoGit(path string) (bool, error) { - r, err := openGoGit(path) + r, err := openHistory(path) if err != nil { return false, err } defer func() { _ = r.Close() }() - shallow, err := r.Storer.Shallow() - return len(shallow) != 0, err + return r.Shallow() } -func openGoGit(path string) (*gogit.Repository, error) { - for _, name := range []string{"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"} { - if os.Getenv(name) != "" { - return nil, fmt.Errorf("gogit backend does not support %s", name) - } +func changeFromHistory(c githistory.Change) change { + return change{ + commit: c.Commit, date: c.Date, subject: c.Subject, + oldOID: c.OldOID, newOID: c.NewOID, path: c.Path, + oldMode: c.OldMode, newMode: c.NewMode, + merge: c.Merge, } - r, err := gogit.PlainOpen(path) - if err != nil { - return nil, err - } - fs := r.Storer.(*filesystem.Storage).Filesystem() - for _, name := range []string{"objects/info/alternates", "info/grafts"} { - f, err := fs.Open(name) - if errors.Is(err, os.ErrNotExist) { - continue - } - if err == nil { - var data []byte - data, err = io.ReadAll(f) - closeErr := f.Close() - if err == nil { - err = closeErr - } - if err == nil && len(strings.TrimSpace(string(data))) > 0 { - err = fmt.Errorf("gogit backend does not support %s", name) - } - } - if err != nil { - _ = r.Close() - return nil, err - } - } - if err := configureGoGit(r); err != nil { - _ = r.Close() - return nil, err +} + +func changeToHistory(c change) githistory.Change { + return githistory.Change{ + Commit: c.commit, Date: c.date, Subject: c.subject, + OldOID: c.oldOID, NewOID: c.newOID, Path: c.path, + OldMode: c.oldMode, NewMode: c.newMode, + Merge: c.merge, } - return r, nil } diff --git a/gogit_history_parallel.go b/gogit_history_parallel.go deleted file mode 100644 index 9c2e4bb..0000000 --- a/gogit_history_parallel.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -var historyWorkers = newOption(1) - -const historyLookahead = 2 - -type historyResult struct { - changes []change - err error -} diff --git a/gogit_history_parallel_test.go b/gogit_history_parallel_test.go index 0499327..8b0a72b 100644 --- a/gogit_history_parallel_test.go +++ b/gogit_history_parallel_test.go @@ -26,16 +26,16 @@ func TestGitFreeParallelHistory(t *testing.T) { date := fmt.Sprintf("2025-01-%02dT12:00:00Z", 13-n) t.Setenv("GIT_AUTHOR_DATE", date) t.Setenv("GIT_COMMITTER_DATE", date) - license := []string{"MIT", testApache}[n%2] + license := []string{mitExpression, testApache}[n%2] commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: "+license+"\n", fmt.Sprintf("Change %d", n)) } git(t, repo, "gc", "--quiet") for _, workers := range []string{"2", "4"} { args := []string{"--history-workers=" + workers, testMmapFlag, "--readers=4"} for _, mode := range []string{"--details", "--monthly"} { - commandArgs := append([]string{"log", repo}, args...) + commandArgs := append([]string{testLogCommand, repo}, args...) got := cliWithoutGit(t, append(commandArgs, mode)...) - want := cli(t, "log", repo, mode) + want := cli(t, testLogCommand, repo, mode) if got != want { t.Fatalf("history differs:\n%s\n%s", got, want) } @@ -59,16 +59,20 @@ func TestGitFreeParallelHistoryMissingTree(t *testing.T) { if err := os.Remove(filepath.Join(repo, ".git", "objects", hash[:2], hash[2:])); err != nil { t.Fatal(err) } + cmd = exec.Command("git", "-C", repo, "cat-file", "-e", hash) + if out, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("tree %s remains readable after removing its loose object:\n%s", hash, out) + } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() - cmd = exec.CommandContext(ctx, os.Args[0], "scan", repo, "--backend=gogit", "--history-workers=4") + cmd = exec.CommandContext(ctx, os.Args[0], testScanCommand, repo, "--backend=gogit", "--history-workers=4") cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "PATH="+t.TempDir()) out, err = cmd.CombinedOutput() if ctx.Err() != nil { t.Fatal("history workers did not terminate") } if err == nil || !strings.Contains(string(out), "object not found") { - t.Fatalf("missing tree accepted: %v\n%s", err, out) + t.Fatalf("missing tree %s accepted: %v\n%s", hash, err, out) } } @@ -77,7 +81,7 @@ func BenchmarkParallelHistory(b *testing.B) { b.Cleanup(func() { *backend, *goGitMmap, *historyWorkers = oldBackend, oldMmap, oldWorkers }) *backend, *goGitMmap = goGitBackend, true for _, root := range benchmarkRepositoryRoots(b) { - for _, workers := range []int{1, 2, 4} { + for _, workers := range []int{1, 2, 4, 8} { *historyWorkers = workers b.Run(fmt.Sprint(workers), func(b *testing.B) { b.ReportAllocs() diff --git a/gogit_history_stream.go b/gogit_history_stream.go index 11604ad..188eeb1 100644 --- a/gogit_history_stream.go +++ b/gogit_history_stream.go @@ -1,329 +1,19 @@ package main import ( - "bufio" - "bytes" "context" - "encoding/hex" - "errors" "fmt" - "io" - "math" - "strings" "sync" - "time" - gogit "github.com/go-git/go-git/v6" + githistory "github.com/git-pkgs/history" + "github.com/git-pkgs/roles" "github.com/go-git/go-git/v6/plumbing" "github.com/go-git/go-git/v6/plumbing/filemode" - "github.com/go-git/go-git/v6/plumbing/object" - "github.com/go-git/go-git/v6/plumbing/object/commitgraph" ) -type historyRootNode struct { - nodes []commitgraph.CommitNode - hashes []plumbing.Hash -} - -func (n *historyRootNode) ID() plumbing.Hash { return plumbing.ZeroHash } -func (n *historyRootNode) Tree() (*object.Tree, error) { - return nil, errors.New("history root has no tree") -} -func (n *historyRootNode) CommitTime() time.Time { return time.Date(9999, 1, 1, 0, 0, 0, 0, time.UTC) } -func (n *historyRootNode) NumParents() int { return len(n.nodes) } -func (n *historyRootNode) ParentNodes() commitgraph.CommitNodeIter { - return &historyRootIter{nodes: n.nodes} -} -func (n *historyRootNode) ParentHashes() []plumbing.Hash { return n.hashes } -func (n *historyRootNode) Generation() uint64 { return math.MaxUint64 } -func (n *historyRootNode) GenerationV2() uint64 { return math.MaxUint64 } -func (n *historyRootNode) Commit() (*object.Commit, error) { - return nil, errors.New("history root has no commit") -} - -func (n *historyRootNode) ParentNode(i int) (commitgraph.CommitNode, error) { - if i < 0 || i >= len(n.nodes) { - return nil, object.ErrParentNotFound - } - return n.nodes[i], nil -} - -type historyRootIter struct { - nodes []commitgraph.CommitNode - next int -} - -func (i *historyRootIter) Next() (commitgraph.CommitNode, error) { - if i.next == len(i.nodes) { - return nil, io.EOF - } - n := i.nodes[i.next] - i.next++ - return n, nil -} - -func (i *historyRootIter) ForEach(visit func(commitgraph.CommitNode) error) error { - for { - n, err := i.Next() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - if err := visit(n); err != nil { - return err - } - } -} - -func (i *historyRootIter) Close() {} - -type shallowCommitNodeIndex struct { - commitgraph.CommitNodeIndex - boundary map[plumbing.Hash]bool -} - -func (i *shallowCommitNodeIndex) Get(hash plumbing.Hash) (commitgraph.CommitNode, error) { - n, err := i.CommitNodeIndex.Get(hash) - if err != nil || !i.boundary[hash] { - return n, err - } - return &shallowCommitNode{CommitNode: n}, nil -} - -type shallowCommitNode struct { - commitgraph.CommitNode -} - -func (n *shallowCommitNode) NumParents() int { return 0 } -func (n *shallowCommitNode) ParentNodes() commitgraph.CommitNodeIter { return &historyRootIter{} } -func (n *shallowCommitNode) ParentNode(int) (commitgraph.CommitNode, error) { - return nil, object.ErrParentNotFound -} -func (n *shallowCommitNode) ParentHashes() []plumbing.Hash { return nil } - -type historyTreeLoad struct { - ready chan struct{} - tree *object.Tree - err error -} - -type historyTreeCache struct { - mu sync.Mutex - trees map[plumbing.Hash]*historyTreeLoad -} - -func newHistoryTreeCache() *historyTreeCache { - return &historyTreeCache{trees: make(map[plumbing.Hash]*historyTreeLoad)} -} - -func (c *historyTreeCache) load(hash plumbing.Hash, load func() (*object.Tree, error)) (*object.Tree, error) { - c.mu.Lock() - entry := c.trees[hash] - if entry == nil { - entry = &historyTreeLoad{ready: make(chan struct{})} - c.trees[hash] = entry - c.mu.Unlock() - entry.tree, entry.err = load() - close(entry.ready) - return entry.tree, entry.err - } - c.mu.Unlock() - <-entry.ready - return entry.tree, entry.err -} - -func (c *historyTreeCache) release(hash plumbing.Hash) { - c.mu.Lock() - delete(c.trees, hash) - c.mu.Unlock() -} - -func walkChangesGoGitStream(r *gogit.Repository, merges bool, visit func(change)) error { - rootHashes, err := historyRoots(r) - if err != nil { - return err - } - if len(rootHashes) == 0 { - return nil - } - shallow, err := r.Storer.Shallow() - if err != nil { - return err - } - boundary := make(map[plumbing.Hash]bool, len(shallow)) - for _, hash := range shallow { - boundary[hash] = true - } - index := &shallowCommitNodeIndex{ - CommitNodeIndex: commitgraph.NewObjectCommitNodeIndex(r.Storer), - boundary: boundary, - } - root := &historyRootNode{} - seen := make(map[plumbing.Hash]bool, len(rootHashes)) - for _, hash := range rootHashes { - if seen[hash] { - continue - } - seen[hash] = true - node, err := index.Get(hash) - if err != nil { - return err - } - root.nodes = append(root.nodes, node) - root.hashes = append(root.hashes, hash) - } - iter := commitgraph.NewCommitNodeIterDateOrder(root, nil, nil) - defer iter.Close() - if _, err := iter.Next(); err != nil { - return err - } - cache := newHistoryTreeCache() - if *historyWorkers > 1 { - return visitHistoryStreamParallel(iter, cache, merges, visit, *historyWorkers) - } - for { - node, err := iter.Next() - if err == io.EOF { - return nil - } - if err != nil { - return err - } - if merges || node.NumParents() < 2 { - if err := visitCommitNodeChanges(context.Background(), node, cache, visit); err != nil { - return err - } - } - cache.release(node.ID()) - } -} - -func visitCommitNodeChanges(ctx context.Context, node commitgraph.CommitNode, cache *historyTreeCache, visit func(change)) error { - commit, err := node.Commit() - if err != nil { - return err - } - to, err := cache.load(node.ID(), node.Tree) - if err != nil { - return err - } - var from *object.Tree - parents := node.ParentHashes() - if len(parents) > 0 { - parent, err := node.ParentNode(0) - if err != nil { - return err - } - from, err = cache.load(parents[0], parent.Tree) - if err != nil { - return err - } - } - changes, err := object.DiffTreeContext(ctx, from, to) - if err != nil { - return err - } - hash := node.ID().String() - zero := strings.Repeat("0", len(hash)) - for _, entry := range changes { - path := entry.To.Name - if path == "" { - path = entry.From.Name - } - oldOID, newOID := entry.From.TreeEntry.Hash.String(), entry.To.TreeEntry.Hash.String() - if entry.From.TreeEntry.Mode == 0 { - oldOID = zero - } - if entry.To.TreeEntry.Mode == 0 { - newOID = zero - } - visit(change{ - commit: hash, date: commit.Author.When.Format(time.RFC3339), subject: commitSubject(commit.Message), - path: path, oldOID: oldOID, newOID: newOID, - oldMode: fmt.Sprintf("%06o", entry.From.TreeEntry.Mode), newMode: fmt.Sprintf("%06o", entry.To.TreeEntry.Mode), - merge: len(parents) > 1, - }) - } - return nil -} - -type historyStreamTask struct { - node commitgraph.CommitNode - skip bool - result chan historyResult -} - -func visitHistoryStreamParallel(iter commitgraph.CommitNodeIter, cache *historyTreeCache, merges bool, visit func(change), workers int) error { - ctx, cancel := context.WithCancel(context.Background()) - tasks := make(chan historyStreamTask, workers) - var wg sync.WaitGroup - defer func() { - cancel() - close(tasks) - wg.Wait() - }() - for range workers { - wg.Go(func() { - for task := range tasks { - var result historyResult - if !task.skip { - result.err = visitCommitNodeChanges(ctx, task.node, cache, func(c change) { - result.changes = append(result.changes, c) - }) - } - task.result <- result - } - }) - } - window := historyLookahead * workers - pending := make([]historyStreamTask, 0, window) - done := false - fill := func() error { - for len(pending) < window && !done { - node, err := iter.Next() - if err == io.EOF { - done = true - break - } - if err != nil { - return err - } - task := historyStreamTask{ - node: node, - skip: !merges && node.NumParents() >= 2, - result: make(chan historyResult, 1), - } - pending = append(pending, task) - tasks <- task - } - return nil - } - if err := fill(); err != nil { - return err - } - for len(pending) > 0 { - task := pending[0] - result := <-task.result - if result.err != nil { - return result.err - } - for _, c := range result.changes { - visit(c) - } - cache.release(task.node.ID()) - pending = pending[1:] - if err := fill(); err != nil { - return err - } - } - return nil -} - type legalTreeKey struct { - hash plumbing.Hash - inherited bool + hash plumbing.Hash + state roles.StateKey } type legalTreeCollector struct { @@ -332,7 +22,11 @@ type legalTreeCollector struct { legal map[string]bool } -const legalTreeBatchSize = 48 +const ( + legalTreeBatchSize = 48 + legalTreeBatchesPerWorker = 2 + gitFileTypeMask = 0o170000 +) type legalTreeBatch struct { hashes [legalTreeBatchSize]plumbing.Hash @@ -362,238 +56,42 @@ func (c *legalTreeCollector) add(hash plumbing.Hash) { c.mu.Unlock() } -var rawObjectReaderPool = sync.Pool{ - New: func() any { return bufio.NewReaderSize(nil, 8<<10) }, -} - -var legalFileNamePrefixes = [...]string{ - "licenses", - "license", - "licences", - "licence", - "copying", - "mit-license", - "copyright", - "unlicense", - "notices", - "notice", -} - func legalBlobsGoGit(path string) (map[string]bool, error) { - r, err := openGoGit(path) - if err != nil { - return nil, err - } - defer func() { _ = r.Close() }() - roots, err := historyRoots(r) - if err != nil { - return nil, err - } - shallow, err := r.Storer.Shallow() + repo, err := openHistory(path) if err != nil { return nil, err } - boundary := make(map[plumbing.Hash]bool, len(shallow)) - for _, hash := range shallow { - boundary[hash] = true - } + defer func() { _ = repo.Close() }() collector := newLegalTreeCollector() if *historyWorkers <= 1 { - err = visitCommitTrees(r, roots, boundary, func(hash plumbing.Hash) error { - return collectLegalTreeRaw(r, hash, false, collector) + err = repo.VisitCommitTrees(func(hash plumbing.Hash) error { + return collectLegalTreeRaw(repo, hash, roles.RootState(), collector) }) return collector.legal, err } - err = collectLegalTreesParallel(r, roots, boundary, collector, *historyWorkers-1) + err = collectLegalTreesParallel(repo, collector, *historyWorkers-1) return collector.legal, err } -func visitCommitTrees(r *gogit.Repository, roots []plumbing.Hash, boundary map[plumbing.Hash]bool, visit func(plumbing.Hash) error) error { - seen := make(map[plumbing.Hash]struct{}) - stack := append([]plumbing.Hash(nil), roots...) - for len(stack) > 0 { - last := len(stack) - 1 - hash := stack[last] - stack = stack[:last] - if _, ok := seen[hash]; ok { - continue - } - tree, err := readCommitTreeAndParents(r, hash, !boundary[hash], &stack) - if err != nil { - return err - } - seen[hash] = struct{}{} - if err := visit(tree); err != nil { - return err - } - } - return nil -} - -func readCommitTreeAndParents(r *gogit.Repository, hash plumbing.Hash, includeParents bool, parents *[]plumbing.Hash) (tree plumbing.Hash, resultErr error) { - encoded, err := r.Storer.EncodedObject(plumbing.CommitObject, hash) - if err != nil { - return tree, err - } - if encoded.Type() != plumbing.CommitObject { - return tree, fmt.Errorf("object %s is %s, want commit", hash, encoded.Type()) - } - if memory, ok := encoded.(*plumbing.MemoryObject); ok { - writer := commitLinksWriter{hash: hash, includeParents: includeParents, parents: parents} - _, err := memory.WriteTo(&writer) - return writer.tree, err - } - reader, err := encoded.Reader() - if err != nil { - return tree, err - } - defer func() { - if err := reader.Close(); resultErr == nil { - resultErr = err - } - }() - buffer := rawObjectReaderPool.Get().(*bufio.Reader) - buffer.Reset(reader) - defer func() { - buffer.Reset(nil) - rawObjectReaderPool.Put(buffer) - }() - - line, readErr := buffer.ReadSlice('\n') - if readErr != nil && (readErr != io.EOF || len(line) == 0) { - return tree, fmt.Errorf("commit %s tree: %w", hash, readErr) - } - line = bytes.TrimSuffix(line, []byte{'\n'}) - value, ok := bytes.CutPrefix(line, []byte("tree ")) - if !ok { - return tree, fmt.Errorf("commit %s has no leading tree header", hash) - } - tree, err = parseCommitObjectID(value, hash.Size()) - if err != nil { - return tree, fmt.Errorf("commit %s tree: %w", hash, err) - } - if readErr == io.EOF { - return tree, nil - } - - for { - line, readErr = buffer.ReadSlice('\n') - if readErr != nil && (readErr != io.EOF || len(line) == 0) { - return tree, fmt.Errorf("commit %s parent: %w", hash, readErr) - } - line = bytes.TrimSuffix(line, []byte{'\n'}) - value, ok = bytes.CutPrefix(line, []byte("parent ")) - if !ok { - return tree, nil - } - parent, err := parseCommitObjectID(value, hash.Size()) - if err != nil { - return tree, fmt.Errorf("commit %s parent: %w", hash, err) - } - if includeParents { - *parents = append(*parents, parent) - } - if readErr == io.EOF { - return tree, nil - } - } -} - -type commitLinksWriter struct { - hash plumbing.Hash - tree plumbing.Hash - includeParents bool - parents *[]plumbing.Hash -} - -func (w *commitLinksWriter) Write(data []byte) (int, error) { - line, rest, _ := bytes.Cut(data, []byte{'\n'}) - value, ok := bytes.CutPrefix(line, []byte("tree ")) - if !ok { - return 0, fmt.Errorf("commit %s has no leading tree header", w.hash) - } - tree, err := parseCommitObjectID(value, w.hash.Size()) - if err != nil { - return 0, fmt.Errorf("commit %s tree: %w", w.hash, err) - } - w.tree = tree - for len(rest) > 0 { - line, next, _ := bytes.Cut(rest, []byte{'\n'}) - value, ok = bytes.CutPrefix(line, []byte("parent ")) - if !ok { - break - } - parent, err := parseCommitObjectID(value, w.hash.Size()) - if err != nil { - return 0, fmt.Errorf("commit %s parent: %w", w.hash, err) - } - if w.includeParents { - *w.parents = append(*w.parents, parent) - } - rest = next - } - return len(data), nil -} - -func parseCommitObjectID(value []byte, size int) (plumbing.Hash, error) { - if len(value) != hex.EncodedLen(size) { - return plumbing.ZeroHash, fmt.Errorf("invalid object ID %q", value) - } - var decoded objectID - if _, err := hex.Decode(decoded[:size], value); err != nil { - return plumbing.ZeroHash, fmt.Errorf("invalid object ID %q: %w", value, err) - } - hash, ok := plumbing.FromBytes(decoded[:size]) - if !ok { - return plumbing.ZeroHash, fmt.Errorf("invalid object ID %q", value) - } - return hash, nil -} - -func collectLegalTreesParallel(r *gogit.Repository, roots []plumbing.Hash, boundary map[plumbing.Hash]bool, collector *legalTreeCollector, workers int) error { - ctx, cancel := context.WithCancel(context.Background()) - tasks := make(chan legalTreeBatch, workers*2) +func collectLegalTreesParallel(repo *githistory.Repo, collector *legalTreeCollector, workers int) error { + ctx, cancel := context.WithCancelCause(context.Background()) + defer cancel(nil) + tasks := make(chan legalTreeBatch, workers*legalTreeBatchesPerWorker) var wg sync.WaitGroup - var firstErr error - var errMu sync.Mutex for range workers { - wg.Go(func() { - for { - select { - case <-ctx.Done(): - return - case batch, ok := <-tasks: - if !ok { - return - } - for _, hash := range batch.hashes[:batch.count] { - if err := collectLegalTreeRaw(r, hash, false, collector); err != nil { - errMu.Lock() - if firstErr == nil { - firstErr = err - cancel() - } - errMu.Unlock() - return - } - } - } - } - }) + wg.Go(func() { collectLegalTreeBatches(ctx, cancel, repo, tasks, collector) }) } var batch legalTreeBatch enqueue := func() error { select { case <-ctx.Done(): - errMu.Lock() - defer errMu.Unlock() - return firstErr + return context.Cause(ctx) case tasks <- batch: batch.count = 0 return nil } } - err := visitCommitTrees(r, roots, boundary, func(hash plumbing.Hash) error { + err := repo.VisitCommitTrees(func(hash plumbing.Hash) error { batch.hashes[batch.count] = hash batch.count++ if batch.count == len(batch.hashes) { @@ -606,199 +104,81 @@ func collectLegalTreesParallel(r *gogit.Repository, roots []plumbing.Hash, bound } close(tasks) wg.Wait() - cancel() if err != nil { return err } - return firstErr + return context.Cause(ctx) } -func collectLegalTreeRaw(r *gogit.Repository, hash plumbing.Hash, inherited bool, collector *legalTreeCollector) (resultErr error) { - key := legalTreeKey{hash: hash, inherited: inherited} - if !collector.claim(key) { - return nil - } - object, err := r.Storer.EncodedObject(plumbing.TreeObject, hash) - if err != nil { - return err - } - if object.Type() != plumbing.TreeObject { - return fmt.Errorf("object %s is %s, want tree", hash, object.Type()) - } - if memory, ok := object.(*plumbing.MemoryObject); ok { - writer := legalTreeWriter{repository: r, hash: hash, inherited: inherited, collector: collector} - _, err := memory.WriteTo(writer) - return err - } - reader, err := object.Reader() - if err != nil { - return err - } - defer func() { - if err := reader.Close(); resultErr == nil { - resultErr = err - } - }() - buffer := rawObjectReaderPool.Get().(*bufio.Reader) - buffer.Reset(reader) - defer func() { - buffer.Reset(nil) - rawObjectReaderPool.Put(buffer) - }() +func collectLegalTreeBatches(ctx context.Context, fail context.CancelCauseFunc, r *githistory.Repo, tasks <-chan legalTreeBatch, collector *legalTreeCollector) { for { - modeBytes, err := buffer.ReadSlice(' ') - if err == io.EOF && len(modeBytes) == 0 { - return nil - } - if err != nil { - return fmt.Errorf("tree %s mode: %w", hash, err) - } - mode, err := filemode.FromBytes(modeBytes[:len(modeBytes)-1]) - if err != nil { - return fmt.Errorf("tree %s mode: %w", hash, err) - } - mode = canonicalLegalTreeMode(mode) - name, err := buffer.ReadSlice(0) - if err != nil { - return fmt.Errorf("tree %s name: %w", hash, err) - } - name = name[:len(name)-1] - if len(name) == 0 { - return fmt.Errorf("tree %s has an empty filename", hash) - } - oidBytes, err := buffer.Peek(hash.Size()) - if err != nil { - return fmt.Errorf("tree %s object ID: %w", hash, err) - } - oid, ok := plumbing.FromBytes(oidBytes) - if !ok { - return fmt.Errorf("tree %s has an invalid object ID", hash) - } - if _, err := buffer.Discard(hash.Size()); err != nil { - return fmt.Errorf("tree %s object ID: %w", hash, err) - } - switch mode { - case filemode.Dir: - childLegal := inherited || isLegalDirectoryBytes(name) - if err := collectLegalTreeRaw(r, oid, childLegal, collector); err != nil { - return err + select { + case <-ctx.Done(): + return + case batch, ok := <-tasks: + if !ok { + return } - case filemode.Regular, filemode.Deprecated, filemode.Executable: - if inherited || isLegalFileNameBytes(name) { - collector.add(oid) + if err := collectLegalTreeBatch(r, batch, collector); err != nil { + fail(err) + return } - case filemode.Symlink, filemode.Submodule: - default: - return fmt.Errorf("unsupported mode %s for %q", mode, name) } } } -type legalTreeWriter struct { - repository *gogit.Repository - hash plumbing.Hash - inherited bool - collector *legalTreeCollector +func collectLegalTreeBatch(r *githistory.Repo, batch legalTreeBatch, collector *legalTreeCollector) error { + for _, hash := range batch.hashes[:batch.count] { + if err := collectLegalTreeRaw(r, hash, roles.RootState(), collector); err != nil { + return err + } + } + return nil } -func (w legalTreeWriter) Write(data []byte) (int, error) { - size := len(data) - for len(data) > 0 { - modeEnd := bytes.IndexByte(data, ' ') - if modeEnd < 0 { - return 0, fmt.Errorf("tree %s mode: %w", w.hash, io.ErrUnexpectedEOF) - } - mode, err := filemode.FromBytes(data[:modeEnd]) - if err != nil { - return 0, fmt.Errorf("tree %s mode: %w", w.hash, err) - } - mode = canonicalLegalTreeMode(mode) - data = data[modeEnd+1:] - nameEnd := bytes.IndexByte(data, 0) - if nameEnd < 0 { - return 0, fmt.Errorf("tree %s name: %w", w.hash, io.ErrUnexpectedEOF) - } - name := data[:nameEnd] - if len(name) == 0 { - return 0, fmt.Errorf("tree %s has an empty filename", w.hash) - } - data = data[nameEnd+1:] - if len(data) < w.hash.Size() { - return 0, fmt.Errorf("tree %s object ID: %w", w.hash, io.ErrUnexpectedEOF) - } - oid, ok := plumbing.FromBytes(data[:w.hash.Size()]) - if !ok { - return 0, fmt.Errorf("tree %s has an invalid object ID", w.hash) - } - data = data[w.hash.Size():] +func collectLegalTreeRaw(r *githistory.Repo, hash plumbing.Hash, state roles.State, collector *legalTreeCollector) error { + key := legalTreeKey{hash: hash, state: state.Key()} + if !collector.claim(key) { + return nil + } + return r.WalkTreeEntries(hash, func(entry githistory.TreeEntry) error { + mode := canonicalLegalTreeMode(entry.Mode) switch mode { case filemode.Dir: - childLegal := w.inherited || isLegalDirectoryBytes(name) - if err := collectLegalTreeRaw(w.repository, oid, childLegal, w.collector); err != nil { - return 0, err + child, err := state.Enter(entry.Name) + if err != nil { + return fmt.Errorf("enter tree %q: %w", entry.Name, err) } + return collectLegalTreeRaw(r, entry.Hash, child, collector) case filemode.Regular, filemode.Deprecated, filemode.Executable: - if w.inherited || isLegalFileNameBytes(name) { - w.collector.add(oid) + fileRoles, err := state.Match(entry.Name) + if err != nil { + return fmt.Errorf("match file %q: %w", entry.Name, err) } + if fileRoles.Has(roles.Legal) { + collector.add(entry.Hash) + } + return nil case filemode.Symlink, filemode.Submodule: + return nil default: - return 0, fmt.Errorf("unsupported mode %s for %q", mode, name) + return fmt.Errorf("unsupported mode %s for %q", mode, entry.Name) } - } - return size, nil + }) } func canonicalLegalTreeMode(mode filemode.FileMode) filemode.FileMode { - switch mode & 0o170000 { - case 0o040000: + switch mode & gitFileTypeMask { + case filemode.Dir: return filemode.Dir - case 0o100000: + case filemode.Regular & gitFileTypeMask: if mode&0o111 != 0 { return filemode.Executable } return filemode.Regular - case 0o120000: + case filemode.Symlink: return filemode.Symlink default: return filemode.Submodule } } - -func isLegalDirectoryBytes(name []byte) bool { - return equalFoldASCII(name, "license") || - equalFoldASCII(name, "licenses") || - equalFoldASCII(name, "licence") || - equalFoldASCII(name, "licences") -} - -func isLegalFileNameBytes(name []byte) bool { - for _, prefix := range legalFileNamePrefixes { - if len(name) < len(prefix) || !equalFoldASCII(name[:len(prefix)], prefix) { - continue - } - if len(name) == len(prefix) { - return true - } - switch name[len(prefix)] { - case '.', '-', '_': - return true - } - } - return false -} - -func equalFoldASCII(input []byte, pattern string) bool { - if len(input) != len(pattern) { - return false - } - for index, char := range input { - if char >= 'A' && char <= 'Z' { - char += 'a' - 'A' - } - if char != pattern[index] { - return false - } - } - return true -} diff --git a/gogit_history_test.go b/gogit_history_test.go index b0a1a77..dc832bf 100644 --- a/gogit_history_test.go +++ b/gogit_history_test.go @@ -5,12 +5,8 @@ import ( "os" "os/exec" "path/filepath" - "slices" "strings" "testing" - - "github.com/go-git/go-git/v6/plumbing" - "github.com/go-git/go-git/v6/plumbing/object" ) func cliWithoutGit(t *testing.T, args ...string) string { @@ -26,46 +22,6 @@ func cliWithoutGit(t *testing.T, args ...string) string { return string(out) } -func TestRawCommitLinksMatchDecodedCommits(t *testing.T) { - for _, format := range []string{testSHA1, testSHA256} { - t.Run(format, func(t *testing.T) { - repo := repository(t, "--object-format="+format) - commitFile(t, repo, "base.txt", "base\n", "Base") - git(t, repo, "switch", "-c", "side") - commitFile(t, repo, "side.txt", "side\n", "Side") - git(t, repo, "switch", "main") - commitFile(t, repo, "main.txt", "main\n", "Main") - git(t, repo, "merge", "--no-ff", "-m", "Merge", "side") - git(t, repo, "gc", "--quiet") - - r, err := openGoGit(repo) - if err != nil { - t.Fatal(err) - } - defer func() { _ = r.Close() }() - iter, err := r.CommitObjects() - if err != nil { - t.Fatal(err) - } - defer iter.Close() - err = iter.ForEach(func(commit *object.Commit) error { - var parents []plumbing.Hash - tree, err := readCommitTreeAndParents(r, commit.Hash, true, &parents) - if err != nil { - return err - } - if tree != commit.TreeHash || !slices.Equal(parents, commit.ParentHashes) { - t.Fatalf("commit %s links differ", commit.Hash) - } - return nil - }) - if err != nil { - t.Fatal(err) - } - }) - } -} - func TestGitFreePackedHistory(t *testing.T) { for _, format := range []string{"sha1", "sha256"} { t.Run(format, func(t *testing.T) { @@ -74,12 +30,12 @@ func TestGitFreePackedHistory(t *testing.T) { commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: Apache-2.0\n", "Change license") commitFile(t, repo, testLegalPath(), "SPDX-License-Identifier: MIT\n", "Add vendor notice") git(t, repo, "gc", "--quiet") - want := cli(t, "log", repo, "--details") - got := cliWithoutGit(t, "log", repo, "--details") + want := cli(t, testLogCommand, repo, "--details") + got := cliWithoutGit(t, testLogCommand, repo, "--details") if got != want { t.Fatalf("history differs:\nwant:\n%s\ngot:\n%s", want, got) } - out := cliWithoutGit(t, "scan", repo) + out := cliWithoutGit(t, testScanCommand, repo) if metric(t, out, "blobs with hits") != 2 { t.Fatal(out) } @@ -104,17 +60,43 @@ func TestGitFreeMergeLegalPathAndTag(t *testing.T) { git(t, repo, "tag", "-a", "tag-only", "-m", "Tagged orphan") git(t, repo, "switch", "main") git(t, repo, "branch", "-D", "tagged") - args := []string{"scan", repo, "--max-blob-size=64", "--max-legal-blob-size=1024"} + args := []string{testScanCommand, repo, "--max-blob-size=64", "--max-legal-blob-size=1024"} out := cliWithoutGit(t, args...) if metric(t, out, "blobs with hits") != 2 || metric(t, out, "skipped: size") != 0 { t.Fatal(out) } - args = []string{"log", repo, "--max-blob-size=64", "--max-legal-blob-size=1024", "--monthly"} + args = []string{testLogCommand, repo, "--max-blob-size=64", "--max-legal-blob-size=1024", "--monthly"} if got, want := cliWithoutGit(t, args...), cli(t, args...); got != want { t.Fatalf("monthly counts differ:\n%s\n%s", got, want) } } +func TestGitFreeSharedSubtreeUsesRolesState(t *testing.T) { + repo := repository(t) + content := "SPDX-License-Identifier: MIT\n" + strings.Repeat("x\n", 100) + commitFile(t, repo, "docs/component.txt", content, "Add ordinary copy") + commitFile(t, repo, "licenses/component.txt", content, "Add legal copy") + + var tree string + for _, directory := range []string{"docs", "licenses"} { + cmd := exec.Command("git", "-C", repo, "rev-parse", "HEAD:"+directory) + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + current := strings.TrimSpace(string(out)) + if tree != "" && current != tree { + t.Fatalf("tree for %s = %s, want shared tree %s", directory, current, tree) + } + tree = current + } + + out := cliWithoutGit(t, testScanCommand, repo, "--max-blob-size=64", "--max-legal-blob-size=1024") + if metric(t, out, "blobs with hits") != 1 || metric(t, out, "skipped: size") != 0 { + t.Fatalf("shared subtree lost its legal occurrence:\n%s", out) + } +} + func TestGitFreeBareWorktreeAndShallow(t *testing.T) { repo := repository(t) commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: MIT\n", "Add license") @@ -126,8 +108,8 @@ func TestGitFreeBareWorktreeAndShallow(t *testing.T) { shallow := filepath.Join(t.TempDir(), "shallow") git(t, repo, "clone", "--depth=1", "file://"+repo, shallow) for _, path := range []string{bare, worktree, shallow} { - want := cli(t, "log", path, "--details") - got := cliWithoutGit(t, "log", path, "--details") + want := cli(t, testLogCommand, path, "--details") + got := cliWithoutGit(t, testLogCommand, path, "--details") if got != want { t.Fatalf("history for %s differs:\nwant:\n%s\ngot:\n%s", path, want, got) } @@ -136,24 +118,30 @@ func TestGitFreeBareWorktreeAndShallow(t *testing.T) { func TestGitFreeEmptyRepository(t *testing.T) { repo := repository(t) - if out := cliWithoutGit(t, "scan", repo); metric(t, out, "blobs seen") != 0 { + if out := cliWithoutGit(t, testScanCommand, repo); metric(t, out, "blobs seen") != 0 { t.Fatal(out) } - if out := cliWithoutGit(t, "log", repo); !strings.Contains(out, "root: 0 commits") { + if out := cliWithoutGit(t, testLogCommand, repo); !strings.Contains(out, "root: 0 commits") { t.Fatal(out) } } -func TestGitFreeRejectsAlternateObjectStore(t *testing.T) { +func TestGitFreeAlternateObjectStore(t *testing.T) { repo := repository(t) commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: MIT\n", "Add license") + commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: Apache-2.0\n", "Change license") + git(t, repo, "gc", "--quiet") shared := filepath.Join(t.TempDir(), "shared") git(t, repo, "clone", "--shared", repo, shared) - cmd := exec.Command(os.Args[0], "scan", shared, "--backend=gogit") - cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "GOMAXPROCS=2", "PATH="+t.TempDir()) - out, err := cmd.CombinedOutput() - if err == nil || !strings.Contains(string(out), "does not support objects/info/alternates") { - t.Fatalf("alternate object store must not produce an empty success: %v\n%s", err, out) + want := cliWithoutGit(t, testScanCommand, repo) + got := cliWithoutGit(t, testScanCommand, shared) + for _, name := range []string{ + "blobs seen", "blobs matched", "blobs with hits", + "skipped: size", "skipped: binary", "skipped: error", + } { + if gotMetric, wantMetric := metric(t, got, name), metric(t, want, name); gotMetric != wantMetric { + t.Fatalf("%s=%d, want %d\nsource:\n%s\nshared:\n%s", name, gotMetric, wantMetric, want, got) + } } } @@ -162,10 +150,10 @@ func TestGitFreeHistoryWithClockSkew(t *testing.T) { for i, date := range []string{"2030-01-01T12:00:00Z", "2020-01-01T12:00:00Z", "2025-01-01T12:00:00Z"} { t.Setenv("GIT_AUTHOR_DATE", date) t.Setenv("GIT_COMMITTER_DATE", date) - content := []string{"MIT", "Apache-2.0", "BSD-3-Clause"}[i] + content := []string{mitExpression, testApache, "BSD-3-Clause"}[i] commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: "+content+"\n", "License "+content) } - if got, want := cliWithoutGit(t, "log", repo, "--details"), cli(t, "log", repo, "--details"); got != want { + if got, want := cliWithoutGit(t, testLogCommand, repo, "--details"), cli(t, testLogCommand, repo, "--details"); got != want { t.Fatalf("clock-skewed history differs:\n%s\n%s", got, want) } } @@ -198,7 +186,7 @@ func TestGitFreeScanRejectsCorruptBlobChecksum(t *testing.T) { t.Fatal(err) } for _, klauspost := range []bool{false, true} { - cmd = exec.Command(os.Args[0], "scan", repo, "--backend=gogit", "--readers=4", "--gogit-memory-index", fmt.Sprintf("--gogit-klauspost-zlib=%t", klauspost)) + cmd = exec.Command(os.Args[0], testScanCommand, repo, "--backend=gogit", "--readers=4", "--gogit-memory-index", fmt.Sprintf("--gogit-klauspost-zlib=%t", klauspost)) cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "GOMAXPROCS=2", "PATH="+t.TempDir()) out, err := cmd.CombinedOutput() if err == nil || !strings.Contains(string(out), "checksum") { diff --git a/gogit_tuning.go b/gogit_tuning.go index f5f4474..92254ed 100644 --- a/gogit_tuning.go +++ b/gogit_tuning.go @@ -1,257 +1,57 @@ package main import ( - "context" - "fmt" - "io" - "path/filepath" - "sync" + "runtime" - "github.com/go-git/go-billy/v6/osfs" - gogit "github.com/go-git/go-git/v6" - "github.com/go-git/go-git/v6/plumbing" - "github.com/go-git/go-git/v6/plumbing/cache" - "github.com/go-git/go-git/v6/storage/filesystem" - "github.com/go-git/go-git/v6/storage/filesystem/dotgit" + githistory "github.com/git-pkgs/history" ) -const ( - defaultGoGitObjectBuffer = 64 - defaultGoGitObjectBatch = 64 -) +const defaultHistoryWorkers = 4 + +var defaultGoGitTuning = githistory.DefaultTuning() var ( - goGitMemoryIndex = newOption(false) - goGitMmap = newOption(false) - goGitCacheBytes = newOption(uint64(cache.DefaultMaxSize)) - goGitCacheShards = newOption(1) - goGitObjectInfos = newOption(true) - goGitObjectBuffer = newOption(defaultGoGitObjectBuffer) - goGitObjectBatch = newOption(defaultGoGitObjectBatch) + goGitMemoryIndex = newOption(defaultGoGitTuning.MemoryIndex) + goGitMmap = newOption(defaultGoGitTuning.Mmap) + goGitCacheBytes = newOption(defaultGoGitTuning.CacheBytes) + goGitCacheShards = newOption(defaultGoGitTuning.CacheShards) + goGitKlauspostZlib = newOption(defaultGoGitTuning.KlauspostZlib) + historyWorkers = newOption(min(defaultHistoryWorkers, runtime.GOMAXPROCS(0))) ) -type goGitObjectTask struct { - info filesystem.ObjectInfo - oid string -} - -func configureGoGit(r *gogit.Repository) error { - if !*goGitMemoryIndex && !*goGitMmap && *goGitCacheBytes == uint64(cache.DefaultMaxSize) && *goGitCacheShards <= 1 { - return nil +func goGitTuning() githistory.Tuning { + return githistory.Tuning{ + MemoryIndex: *goGitMemoryIndex, + Mmap: *goGitMmap, + CacheBytes: *goGitCacheBytes, + CacheShards: *goGitCacheShards, + KlauspostZlib: *goGitKlauspostZlib, } - fs := r.Storer.(*filesystem.Storage).Filesystem() - if *goGitMmap { - objects, err := fs.Chroot("objects") - if err != nil { - return err - } - fs = dotgit.NewRepositoryFilesystem( - osfs.New(fs.Root(), osfs.WithBoundOS(), osfs.WithMmap()), - osfs.New(filepath.Dir(objects.Root()), osfs.WithBoundOS(), osfs.WithMmap()), - ) - } - if err := r.Close(); err != nil { - return err - } - objectCache := cache.Object(cache.NewObjectLRU(cache.FileSize(*goGitCacheBytes))) - if *goGitCacheShards > 1 { - objectCache = cache.NewShardedObjectLRU(cache.FileSize(*goGitCacheBytes), *goGitCacheShards) - } - r.Storer = filesystem.NewStorageWithOptions(fs, objectCache, filesystem.Options{ - UseInMemoryIdx: *goGitMemoryIndex, - }) - return nil } -func feedBlobsGoGitParallel(repo string, jobs chan<- job, skip func(string), limit func(string) int64, workers int) (int, error) { - if !*goGitObjectInfos { - return feedBlobsGoGitParallelObjects(repo, jobs, skip, limit, workers) - } - return feedBlobsGoGitParallelObjectInfos(repo, jobs, skip, limit, workers) +func openHistory(path string) (*githistory.Repo, error) { + return githistory.OpenWithOptions(path, githistory.OpenOptions{Tuning: goGitTuning()}) } -func feedBlobsGoGitParallelObjectInfos(repo string, jobs chan<- job, skip func(string), limit func(string) int64, workers int) (int, error) { - r, err := openGoGit(repo) - if err != nil { - return 0, err - } - defer func() { _ = r.Close() }() - storage, ok := r.Storer.(*filesystem.Storage) - if !ok { - return 0, fmt.Errorf("go-git filesystem storage required") - } - iter, err := storage.IterObjectInfos(plumbing.BlobObject) - if err != nil { - return 0, err - } - defer iter.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - batchSize := max(1, *goGitObjectBatch) - bufferedBatches := (*goGitObjectBuffer + batchSize - 1) / batchSize - objects := make(chan []goGitObjectTask, bufferedBatches) - var wg sync.WaitGroup - var once sync.Once - var readErr error - for range workers { - wg.Go(func() { - reader := storage.NewObjectInfoReader() - defer func() { - if err := reader.Close(); err != nil { - once.Do(func() { readErr = err; cancel() }) - } - }() - for batch := range objects { - for _, task := range batch { - obj, err := reader.EncodedObject(task.info) - if err != nil { - once.Do(func() { readErr = err; cancel() }) - return - } - if obj.Size() != task.info.Size { - err = fmt.Errorf("object %s size changed during enumeration: %d != %d", task.info.Hash, obj.Size(), task.info.Size) - once.Do(func() { readErr = err; cancel() }) - return - } - data, err := readGoGitBlob(obj) - if err != nil { - once.Do(func() { readErr = err; cancel() }) - return - } - select { - case jobs <- job{oid: task.oid, data: data}: - case <-ctx.Done(): - return - } - } - } - }) +func goGitReaders() int { + if *readers > 0 { + return *readers } - total := 0 - batch := make([]goGitObjectTask, 0, batchSize) -enumerate: - for { - var info filesystem.ObjectInfo - info, err = iter.Next() - if err == io.EOF { - err = nil - break - } - if err != nil { - break - } - total++ - oid := info.Hash.String() - if info.Size > limit(oid) { - skip(oid) - continue - } - batch = append(batch, goGitObjectTask{info: info, oid: oid}) - if len(batch) == batchSize { - select { - case objects <- batch: - batch = make([]goGitObjectTask, 0, batchSize) - case <-ctx.Done(): - break enumerate - } - } - } - if err == nil && len(batch) != 0 { - select { - case objects <- batch: - case <-ctx.Done(): - } - } - close(objects) - wg.Wait() - if readErr != nil { - return total, readErr - } - return total, err + return runtime.GOMAXPROCS(0) } -func feedBlobsGoGitParallelObjects(repo string, jobs chan<- job, skip func(string), limit func(string) int64, workers int) (int, error) { - r, err := openGoGit(repo) +func feedBlobsGoGit(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) { + r, err := openHistory(repo) if err != nil { return 0, err } defer func() { _ = r.Close() }() - iter, err := r.Storer.IterEncodedObjects(plumbing.BlobObject) - if err != nil { - return 0, err - } - defer iter.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - objects := make(chan plumbing.EncodedObject) - var wg sync.WaitGroup - var once sync.Once - var readErr error - for range workers { - wg.Go(func() { - for obj := range objects { - data, err := readGoGitBlob(obj) - if err != nil { - once.Do(func() { readErr = err; cancel() }) - return - } - select { - case jobs <- job{oid: obj.Hash().String(), data: data}: - case <-ctx.Done(): - return - } - } - }) - } - total := 0 -enumerate: - for { - var obj plumbing.EncodedObject - obj, err = iter.Next() - if err == io.EOF { - err = nil - break - } - if err != nil { - break - } - total++ - oid := obj.Hash().String() - if obj.Size() > limit(oid) { - skip(oid) - continue - } - select { - case objects <- obj: - case <-ctx.Done(): - break enumerate - } - } - close(objects) - wg.Wait() - if readErr != nil { - return total, readErr - } - return total, err -} - -func readGoGitBlob(obj plumbing.EncodedObject) ([]byte, error) { - if memory, ok := obj.(interface{ Bytes() []byte }); ok { - data := memory.Bytes() - if int64(len(data)) != obj.Size() { - return nil, fmt.Errorf("memory object size mismatch: %d != %d", len(data), obj.Size()) - } - return data, nil - } - r, err := obj.Reader() - if err != nil { - return nil, err - } - data := make([]byte, obj.Size()) - _, err = io.ReadFull(r, data) - closeErr := r.Close() - if err != nil { - return nil, err - } - return data, closeErr + return r.WalkBlobs(githistory.BlobOptions{ + Workers: goGitReaders(), + Limit: limit, + Skip: skip, + }, func(blob githistory.Blob) error { + jobs <- job{oid: blob.OID, data: blob.Data} + return nil + }) } diff --git a/gogit_tuning_test.go b/gogit_tuning_test.go index 2d41b10..ec0c388 100644 --- a/gogit_tuning_test.go +++ b/gogit_tuning_test.go @@ -4,48 +4,30 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" - "sync" "testing" - - "github.com/go-git/go-git/v6/plumbing" -) - -var ( - benchmarkZlibOnce sync.Once - benchmarkZlibErr error ) -func TestReadGoGitBlobReusesMemoryObjectContent(t *testing.T) { - obj := plumbing.NewMemoryObject(nil) - obj.SetType(plumbing.BlobObject) - if _, err := obj.Write([]byte("blob content")); err != nil { - t.Fatal(err) - } - - got, err := readGoGitBlob(obj) - if err != nil { - t.Fatal(err) - } - if string(got) != "blob content" { - t.Fatalf("content = %q", got) - } - if &got[0] != &obj.Bytes()[0] { - t.Fatal("memory object content was copied") +func TestDefaultGoGitTuning(t *testing.T) { + got := goGitTuning() + if !got.Mmap || !got.KlauspostZlib { + t.Fatalf("default tuning = %+v", got) } } -func TestReadGoGitBlobRejectsMemoryObjectSizeMismatch(t *testing.T) { - obj := plumbing.NewMemoryObject(nil) - obj.SetType(plumbing.BlobObject) - if _, err := obj.Write([]byte("blob content")); err != nil { - t.Fatal(err) +func TestDefaultGoGitReaders(t *testing.T) { + oldReaders := *readers + t.Cleanup(func() { *readers = oldReaders }) + *readers = 0 + if got, want := goGitReaders(), runtime.GOMAXPROCS(0); got != want { + t.Fatalf("readers = %d, want %d", got, want) } - obj.SetSize(obj.Size() + 1) +} - _, err := readGoGitBlob(obj) - if err == nil || !strings.Contains(err.Error(), "memory object size mismatch") { - t.Fatalf("error = %v", err) +func TestDefaultHistoryWorkers(t *testing.T) { + if got, want := *historyWorkers, min(defaultHistoryWorkers, runtime.GOMAXPROCS(0)); got != want { + t.Fatalf("history workers = %d, want %d", got, want) } } @@ -53,7 +35,8 @@ func TestGitFreeTunedPackedScan(t *testing.T) { for _, format := range []string{"sha1", "sha256"} { t.Run(format, func(t *testing.T) { repo := repository(t, "--object-format="+format) - for i := range 300 { + const sourceFiles = 1100 + for i := range sourceFiles { content := fmt.Sprintf("SPDX-License-Identifier: MIT\nunique=%d\n%s", i, strings.Repeat("sample source line\n", 100)) if err := os.WriteFile(filepath.Join(repo, fmt.Sprintf("file-%d.go", i)), []byte(content), 0600); err != nil { t.Fatal(err) @@ -62,13 +45,13 @@ func TestGitFreeTunedPackedScan(t *testing.T) { git(t, repo, "add", ".") commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: Apache-2.0\n", "Add source files") git(t, repo, "repack", "-adq") - want := cli(t, "scan", repo) - if metric(t, want, "blobs seen") != 301 { + want := cli(t, testScanCommand, repo) + if metric(t, want, "blobs seen") != sourceFiles+1 { t.Fatal(want) } for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} { for _, readers := range []string{"1", "2", "4"} { - args := []string{"scan", repo, "--readers=" + readers, mode, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8", "--gogit-object-buffer=64", "--gogit-object-batch=64"} + args := []string{testScanCommand, repo, "--readers=" + readers, mode, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8"} got := cliWithoutGit(t, args...) assertScanMetrics(t, got, want) } @@ -84,7 +67,7 @@ func assertScanMetrics(t *testing.T, got, want string) { t.Fatalf("%s differs:\n%s\n%s", name, got, want) } } - if !strings.Contains(got, "MIT") || !strings.Contains(got, "Apache-2.0") { + if !strings.Contains(got, mitExpression) || !strings.Contains(got, testApache) { t.Fatal(got) } } @@ -102,8 +85,8 @@ func TestGitFreeTunedHistoryLayouts(t *testing.T) { git(t, repo, "clone", "--depth=1", "file://"+repo, shallow) for _, path := range []string{repo, bare, worktree, shallow} { for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} { - got := cliWithoutGit(t, "log", path, "--readers=4", mode, "--gogit-cache-bytes=536870912", "--details") - if want := cli(t, "log", path, "--details"); got != want { + got := cliWithoutGit(t, testLogCommand, path, "--readers=4", mode, "--gogit-cache-bytes=536870912", "--details") + if want := cli(t, testLogCommand, path, "--details"); got != want { t.Fatalf("history differs for %s:\n%s\n%s", path, got, want) } } @@ -184,42 +167,6 @@ func BenchmarkGoGitCacheReaders4(b *testing.B) { } } -func BenchmarkGoGitObjectInfos(b *testing.B) { - if os.Getenv("GITSPDX_BENCH_KLAUSPOST") == "1" { - benchmarkZlibOnce.Do(func() { - *goGitKlauspostZlib = true - benchmarkZlibErr = configureGoGitZlib() - }) - if benchmarkZlibErr != nil { - b.Fatal(benchmarkZlibErr) - } - } - oldBackend, oldReaders, oldMemory, oldCache, oldMmap, oldObjectInfos := *backend, *readers, *goGitMemoryIndex, *goGitCacheBytes, *goGitMmap, *goGitObjectInfos - oldBuffer, oldBatch := *goGitObjectBuffer, *goGitObjectBatch - b.Cleanup(func() { - *backend, *readers, *goGitMemoryIndex, *goGitCacheBytes = oldBackend, oldReaders, oldMemory, oldCache - *goGitMmap, *goGitObjectInfos = oldMmap, oldObjectInfos - *goGitObjectBuffer = oldBuffer - *goGitObjectBatch = oldBatch - }) - *backend, *readers, *goGitMemoryIndex, *goGitCacheBytes, *goGitMmap = goGitBackend, 8, false, 96<<20, true - *goGitObjectBuffer, *goGitObjectBatch = 64, 64 - for _, mode := range []struct { - name string - infos bool - }{ - {name: "eager", infos: false}, - {name: "metadata", infos: true}, - } { - *goGitObjectInfos = mode.infos - for _, root := range benchmarkRepositoryRoots(b) { - b.Run(mode.name+"/"+filepath.Base(filepath.Clean(root)), func(b *testing.B) { - benchmarkTunedFeed(b, root) - }) - } - } -} - func benchmarkTunedFeed(b *testing.B, root string) { b.Helper() b.ReportAllocs() diff --git a/gogit_zlib.go b/gogit_zlib.go deleted file mode 100644 index 0e06f04..0000000 --- a/gogit_zlib.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "fmt" - "io" - - "github.com/go-git/go-git/v6/x/plugin" - gitzlib "github.com/go-git/go-git/v6/x/plugin/zlib" - kpzlib "github.com/klauspost/compress/zlib" -) - -var goGitKlauspostZlib = newOption(false) - -func configureGoGitZlib() error { - if !*goGitKlauspostZlib { - return nil - } - return plugin.Register(plugin.Zlib(), func() plugin.ZlibProvider { - return klauspostZlibProvider{} - }) -} - -type klauspostZlibProvider struct{} - -func (klauspostZlibProvider) NewReader(r io.Reader) (gitzlib.Reader, error) { - reader, err := kpzlib.NewReader(r) - if err != nil { - return nil, err - } - resettable, ok := reader.(gitzlib.Reader) - if !ok { - _ = reader.Close() - return nil, fmt.Errorf("klauspost zlib reader %T does not implement reset", reader) - } - return resettable, nil -} - -func (klauspostZlibProvider) NewWriter(w io.Writer) gitzlib.Writer { - return kpzlib.NewWriter(w) -} diff --git a/history.go b/history.go index 8387bf1..4e82bdd 100644 --- a/history.go +++ b/history.go @@ -1,18 +1,15 @@ package main import ( - "bufio" - "bytes" "context" "encoding/csv" "fmt" - "io" "os" - "os/exec" "sort" "strconv" "strings" + githistory "github.com/git-pkgs/history" "github.com/git-pkgs/licenses" ) @@ -34,77 +31,10 @@ func walkChanges(repo string, merges bool, visit func(change)) error { if *backend == goGitBackend { return walkChangesGoGit(repo, merges, visit) } - mergeMode := "off" - if merges { - mergeMode = "first-parent" - } - cmd := exec.Command("git", "-C", repo, "log", "--all", "--date-order", - "--root", "--no-abbrev", "--raw", "--no-renames", "-z", - "--diff-merges="+mergeMode, "--format=%x01%H%x00%aI%x00%P%x00%s") - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - cmd.Stderr = os.Stderr - if err := cmd.Start(); err != nil { - return err - } - err = readChanges(stdout, visit) - if err != nil { - _ = cmd.Process.Kill() - } - waitErr := cmd.Wait() - if err != nil { - return fmt.Errorf("read history: %w", err) - } - return waitErr -} - -func readChanges(input io.Reader, visit func(change)) error { - r := bufio.NewReaderSize(input, readerBufferSize) - field := func() (string, error) { - s, err := r.ReadString(0) - return strings.TrimSuffix(s, "\x00"), err - } - var c change - for { - token, err := field() - if err == io.EOF && strings.TrimSpace(token) == "" { - return nil - } - if err != nil { - return err - } - token = strings.TrimLeft(token, "\n") - if strings.HasPrefix(token, "\x01") { - c.commit = token[1:] - if c.date, err = field(); err != nil { - return err - } - parents, err := field() - if err != nil { - return err - } - c.merge = strings.Contains(parents, " ") - if c.subject, err = field(); err != nil { - return err - } - continue - } - if !strings.HasPrefix(token, ":") { - continue - } - fields := strings.Fields(token[1:]) - if len(fields) != 5 || c.commit == "" { - return fmt.Errorf("invalid raw change %q", token) - } - c.oldMode, c.newMode = fields[0], fields[1] - c.oldOID, c.newOID = fields[2], fields[3] - if c.path, err = field(); err != nil { - return err - } - visit(c) - } + return githistory.WalkChangesGit(repo, merges, func(c githistory.Change) error { + visit(changeFromHistory(c)) + return nil + }) } func pathGroup(path string) string { @@ -207,6 +137,11 @@ func logCmd(repo string) error { if *monthly { return printMonthly(months) } + printHistoryTotals(totals) + return nil +} + +func printHistoryTotals(totals map[string]*historyTotals) { for _, role := range []string{groupRoot, groupLegal, groupOther} { if *group != groupAll && *group != role { continue @@ -215,15 +150,13 @@ func logCmd(repo string) error { fmt.Printf("%s: %d commits; %d expression changes; %d file additions; %d file deletions; %d incomplete comparisons; %d SPDX declarations added; %d removed\n", role, t.commits, t.expressions, t.additions, t.deletions, t.incomplete, t.spdxAdded, t.spdxRemoved) } - return nil } func isShallow(repo string) (bool, error) { if *backend == goGitBackend { return isShallowGoGit(repo) } - out, err := exec.Command("git", "-C", repo, "rev-parse", "--is-shallow-repository").Output() - return bytes.HasPrefix(out, []byte("true")), err + return githistory.ShallowGit(repo) } func printChange(c change, role, kind string, before, after blobResult) { diff --git a/history_spool.go b/history_spool.go index ab134b1..a23ed1b 100644 --- a/history_spool.go +++ b/history_spool.go @@ -1,117 +1,40 @@ package main -import ( - "bufio" - "encoding/binary" - "fmt" - "io" - "os" -) +import githistory "github.com/git-pkgs/history" type historySpool struct { - file *os.File - writer *bufio.Writer - err error + spool *githistory.Spool } func spoolLogHistory(repo string) (*historySpool, map[string]bool, error) { - file, err := os.CreateTemp("", "git-spdx-history-*") + spool, err := githistory.NewSpool("git-spdx") if err != nil { return nil, nil, err } - spool := &historySpool{file: file, writer: bufio.NewWriterSize(file, readerBufferSize)} + result := &historySpool{spool: spool} legal := make(map[string]bool) err = walkChanges(repo, true, func(c change) { recordLegalChange(legal, c) if !c.merge { - spool.write(c) + spool.Write(changeToHistory(c)) } }) if err == nil { - err = spool.ready() + err = spool.Ready() } if err != nil { - spool.close() + result.close() return nil, nil, err } - return spool, legal, nil -} - -func (s *historySpool) write(c change) { - if s.err != nil { - return - } - for _, field := range [...]string{c.commit, c.date, c.subject, c.oldOID, c.newOID, c.path, c.oldMode, c.newMode} { - if err := writeSpoolString(s.writer, field); err != nil { - s.err = err - return - } - } -} - -func (s *historySpool) ready() error { - if s.err != nil { - return s.err - } - if err := s.writer.Flush(); err != nil { - return err - } - _, err := s.file.Seek(0, io.SeekStart) - return err + return result, legal, nil } func (s *historySpool) replay(visit func(change)) error { - reader := bufio.NewReaderSize(s.file, readerBufferSize) - for { - commit, err := readSpoolString(reader) - if err == io.EOF { - return nil - } - if err != nil { - return err - } - fields := make([]string, 7) - for index := range fields { - fields[index], err = readSpoolString(reader) - if err != nil { - return err - } - } - visit(change{ - commit: commit, date: fields[0], subject: fields[1], - oldOID: fields[2], newOID: fields[3], path: fields[4], - oldMode: fields[5], newMode: fields[6], - }) - } + return s.spool.Replay(func(c githistory.Change) { + visit(changeFromHistory(c)) + }) } func (s *historySpool) close() { - name := s.file.Name() - _ = s.file.Close() - _ = os.Remove(name) -} - -func writeSpoolString(writer io.Writer, value string) error { - var encoded [binary.MaxVarintLen64]byte - size := binary.PutUvarint(encoded[:], uint64(len(value))) - if _, err := writer.Write(encoded[:size]); err != nil { - return err - } - _, err := io.WriteString(writer, value) - return err -} - -func readSpoolString(reader *bufio.Reader) (string, error) { - size, err := binary.ReadUvarint(reader) - if err != nil { - return "", err - } - if size > uint64(int(^uint(0)>>1)) { - return "", fmt.Errorf("history field is too large: %d", size) - } - value := make([]byte, int(size)) - if _, err := io.ReadFull(reader, value); err != nil { - return "", err - } - return string(value), nil + s.spool.Close() } diff --git a/legal_bench_test.go b/legal_bench_test.go index 749fe1e..533b66b 100644 --- a/legal_bench_test.go +++ b/legal_bench_test.go @@ -78,11 +78,11 @@ func BenchmarkCommitObjects(b *testing.B) { b.ReportAllocs() var count int for b.Loop() { - repository, err := openGoGit(root) + repository, err := openHistory(root) if err != nil { b.Fatal(err) } - iter, err := repository.CommitObjects() + iter, err := repository.Repository().CommitObjects() if err != nil { b.Fatal(err) } @@ -113,41 +113,18 @@ func BenchmarkReachableCommitTrees(b *testing.B) { b.ReportAllocs() var count int for b.Loop() { - repository, err := openGoGit(root) + repository, err := openHistory(root) if err != nil { b.Fatal(err) } - roots, err := historyRoots(repository) - if err != nil { - b.Fatal(err) - } - shallow, err := repository.Storer.Shallow() + count = 0 + err = repository.VisitCommitTrees(func(plumbing.Hash) error { + count++ + return nil + }) if err != nil { b.Fatal(err) } - boundary := make(map[plumbing.Hash]bool, len(shallow)) - for _, hash := range shallow { - boundary[hash] = true - } - seen := make(map[plumbing.Hash]struct{}) - stack := append([]plumbing.Hash(nil), roots...) - for len(stack) > 0 { - last := len(stack) - 1 - hash := stack[last] - stack = stack[:last] - if _, ok := seen[hash]; ok { - continue - } - commit, err := repository.CommitObject(hash) - if err != nil { - b.Fatal(err) - } - seen[hash] = struct{}{} - if !boundary[hash] { - stack = append(stack, commit.ParentHashes...) - } - } - count = len(seen) if err := repository.Close(); err != nil { b.Fatal(err) } diff --git a/main.go b/main.go index 3cd5ada..3eb2152 100644 --- a/main.go +++ b/main.go @@ -22,8 +22,6 @@ import ( "time" "github.com/git-pkgs/licenses" - "github.com/go-git/go-git/v6/plumbing" - "github.com/go-git/go-git/v6/plumbing/cache" "github.com/spf13/cobra" ) @@ -35,8 +33,6 @@ const ( initialExpressionCapacity = 128 readerBufferSize = 1 << 16 jobQueueSize = 256 - jobBatchSize = 64 - jobBatchBytes = 8 << 20 catHeaderFields = 3 expressionDisplayLimit = 20 zeroOID = "0000000000000000000000000000000000000000" @@ -134,17 +130,14 @@ func addSharedFlags(cmd *cobra.Command) { flags := cmd.Flags() flags.StringVar(backend, "backend", "git", "backend: git (subprocesses) or gogit (Git-free)") flags.Int64Var(maxBlobSize, "max-blob-size", defaultMaxBlobSize, "skip blobs larger than this many bytes") - flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS for git, 1 for gogit)") + flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS)") flags.Int64Var(legalBlobSize, "max-legal-blob-size", defaultLegalBlobSize, "size limit for blobs used at legal paths") - flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", false, "load pack indexes into memory") - flags.BoolVar(goGitMmap, "gogit-mmap", false, "memory-map read-only pack and index files") - flags.Uint64Var(goGitCacheBytes, "gogit-cache-bytes", uint64(cache.DefaultMaxSize), "go-git object cache capacity in bytes") - flags.IntVar(goGitCacheShards, "gogit-cache-shards", 1, "go-git object cache shards") - flags.BoolVar(goGitObjectInfos, "gogit-object-infos", true, "enumerate go-git object metadata before loading blobs") - flags.IntVar(goGitObjectBuffer, "gogit-object-buffer", defaultGoGitObjectBuffer, "buffered go-git object metadata entries") - flags.IntVar(goGitObjectBatch, "gogit-object-batch", defaultGoGitObjectBatch, "adjacent go-git object metadata entries per reader task") - flags.BoolVar(goGitKlauspostZlib, "gogit-klauspost-zlib", false, "use klauspost zlib for go-git object decompression") - flags.IntVar(historyWorkers, "history-workers", 1, "go-git history workers") + flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", defaultGoGitTuning.MemoryIndex, "load pack indexes into memory") + flags.BoolVar(goGitMmap, "gogit-mmap", defaultGoGitTuning.Mmap, "memory-map read-only pack and index files") + flags.Uint64Var(goGitCacheBytes, "gogit-cache-bytes", defaultGoGitTuning.CacheBytes, "go-git object cache capacity in bytes") + flags.IntVar(goGitCacheShards, "gogit-cache-shards", defaultGoGitTuning.CacheShards, "go-git object cache shards") + flags.BoolVar(goGitKlauspostZlib, "gogit-klauspost-zlib", defaultGoGitTuning.KlauspostZlib, "use klauspost zlib for go-git object decompression") + flags.IntVar(historyWorkers, "history-workers", min(defaultHistoryWorkers, runtime.GOMAXPROCS(0)), "go-git history workers") } func validateOptions(history bool) error { @@ -154,19 +147,13 @@ func validateOptions(history bool) error { if *maxBlobSize < 0 || *legalBlobSize < 0 { return fmt.Errorf("blob size limits must be non-negative") } - if *goGitObjectBuffer < 0 { - return fmt.Errorf("go-git object buffer must be non-negative") - } - if *goGitObjectBatch <= 0 { - return fmt.Errorf("go-git object batch must be positive") - } if history && !slices.Contains([]string{groupAll, groupRoot, groupLegal, groupOther}, *group) { return fmt.Errorf("unknown history group %q", *group) } if history && *monthly && *details { return fmt.Errorf("--monthly and --details cannot be combined") } - return configureGoGitZlib() + return nil } func repositoryArgument(args []string) string { @@ -516,73 +503,6 @@ func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit fun return total, err } -func feedBlobsGoGit(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) { - if *readers > 1 { - return feedBlobsGoGitParallel(repo, jobs, skip, limit, *readers) - } - return feedBlobsGoGitSerial(repo, jobs, skip, limit) -} - -func feedBlobsGoGitSerial(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) { - r, err := openGoGit(repo) - if err != nil { - return 0, err - } - defer func() { _ = r.Close() }() - iter, err := r.Storer.IterEncodedObjects(plumbing.BlobObject) - if err != nil { - return 0, err - } - defer iter.Close() - total := 0 - batch := make([]job, 0, jobBatchSize) - batchBytes := 0 - flush := func() { - for _, j := range batch { - jobs <- j - } - batch = batch[:0] - batchBytes = 0 - } - for { - obj, err := iter.Next() - if err == io.EOF { - flush() - return total, nil - } - if err != nil { - return total, err - } - total++ - oid := obj.Hash().String() - if obj.Size() > limit(oid) { - skip(oid) - continue - } - rd, err := obj.Reader() - if err != nil { - return total, err - } - data := make([]byte, obj.Size()) - _, err = io.ReadFull(rd, data) - closeErr := rd.Close() - if err != nil { - return total, err - } - if closeErr != nil { - return total, closeErr - } - if len(batch) > 0 && batchBytes+len(data) > jobBatchBytes { - flush() - } - batch = append(batch, job{oid: oid, data: data}) - batchBytes += len(data) - if len(batch) == cap(batch) { - flush() - } - } -} - func matchBlob(ctx context.Context, m *licenses.Matcher, data []byte) blobResult { if bytes.IndexByte(data, 0) >= 0 { return blobResult{skipped: skippedBinary} diff --git a/main_test.go b/main_test.go index 5950967..afa1e72 100644 --- a/main_test.go +++ b/main_test.go @@ -17,6 +17,8 @@ import ( const ( mitExpression = "MIT" testGitExecutable = "git" + testLogCommand = "log" + testScanCommand = "scan" ) func TestMain(m *testing.M) { @@ -54,15 +56,15 @@ func TestCLILegalBlobCaps(t *testing.T) { git(t, repo, "rm", "Godeps/LICENSES") git(t, repo, "commit", "-m", "Remove bundle") commitFile(t, repo, "generated.txt", content+"extra\n", "Add unrelated large blob") - for _, backend := range []string{"git", "gogit"} { + for _, backend := range []string{"git", goGitBackend} { t.Run(backend, func(t *testing.T) { - out := cli(t, "scan", repo, "--backend", backend) + out := cli(t, testScanCommand, repo, "--backend", backend) if metric(t, out, "blobs with hits") != 1 || metric(t, out, "skipped: size") != 1 { t.Fatalf("legal alias must lift only its own blob cap:\n%s", out) } }) } - out := cli(t, "scan", repo, "--max-legal-blob-size", "1048576") + out := cli(t, testScanCommand, repo, "--max-legal-blob-size", "1048576") if metric(t, out, "skipped: size") != 2 { t.Fatalf("explicit legal cap must apply:\n%s", out) } @@ -102,7 +104,7 @@ func TestCLIIncompleteComparison(t *testing.T) { repo := repository(t) commitFile(t, repo, "source.go", "// SPDX-License-Identifier: MIT\n", "Add license") commitFile(t, repo, "source.go", content, "Unscannable file") - out := cli(t, "log", repo, "--max-blob-size", "1000", "--details") + out := cli(t, testLogCommand, repo, "--max-blob-size", "1000", "--details") if !strings.Contains(out, "1 incomplete comparisons") || !strings.Contains(out, "0 expression changes") || strings.Contains(out, " - MIT") { t.Fatalf("skipped result must not imply removal:\n%s", out) } @@ -116,15 +118,19 @@ func TestCLIHistoryGroupsAndPaths(t *testing.T) { for _, name := range []string{"LICENSE", "src/component/LICENSE", oddPath, "source.go"} { commitFile(t, repo, name, "// SPDX-License-Identifier: MIT\n", "Add "+strconv.Quote(name)) } - out := cli(t, "log", repo) - for _, want := range []string{"root: 1 commits", "legal: 2 commits", "other: 1 commits"} { - if !strings.Contains(out, want) { - t.Fatalf("missing %q:\n%s", want, out) - } - } - out = cli(t, "log", repo, "--group", "legal", "--details") - if !strings.Contains(out, strconv.Quote(oddPath)) || strings.Contains(out, "[other]") || strings.Contains(out, "root:") { - t.Fatalf("filter or NUL-delimited path handling failed:\n%s", out) + for _, backend := range []string{"git", goGitBackend} { + t.Run(backend, func(t *testing.T) { + out := cli(t, testLogCommand, repo, "--backend="+backend) + for _, want := range []string{"root: 1 commits", "legal: 2 commits", "other: 1 commits"} { + if !strings.Contains(out, want) { + t.Fatalf("missing %q:\n%s", want, out) + } + } + out = cli(t, testLogCommand, repo, "--backend="+backend, "--group", "legal", "--details") + if !strings.Contains(out, strconv.Quote(oddPath)) || strings.Contains(out, "[other]") || strings.Contains(out, "root:") { + t.Fatalf("filter or NUL-delimited path handling failed:\n%s", out) + } + }) } } @@ -143,7 +149,7 @@ func TestCLILogUsesSingleHistoryWalk(t *testing.T) { } copyExecutable(t, os.Args[0], filepath.Join(wrapperDir, wrapperName)) trace := filepath.Join(t.TempDir(), "git.log") - cmd := exec.Command(os.Args[0], "log", repo) + cmd := exec.Command(os.Args[0], testLogCommand, repo) cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "GIT_SPDX_TEST_GIT_WRAPPER=1", @@ -248,7 +254,7 @@ func TestCLISPDXDeclarationWithoutExpressionChange(t *testing.T) { commitFile(t, repo, "source.c", mitNotice, "Add notice") commitFile(t, repo, "source.c", mitNotice+mitNotice, "Repeat notice") commitFile(t, repo, "source.c", "// SPDX-License-Identifier: MIT\n"+mitNotice, "Add explicit declaration") - out := cli(t, "log", repo, "--details") + out := cli(t, testLogCommand, repo, "--details") if strings.Contains(out, "Repeat notice") || !strings.Contains(out, "Add explicit declaration") || !strings.Contains(out, "0 expression changes") || !strings.Contains(out, "1 SPDX declarations added") { t.Fatalf("declarations and repeated notices must be separate from expression changes:\n%s", out) } @@ -259,7 +265,7 @@ func TestCLICorpusCoveredSPDXDeclaration(t *testing.T) { source := "int example(void) { return 0; }\n" commitFile(t, repo, "source.c", source, "Add source") commitFile(t, repo, "source.c", "// SPDX-License-Identifier: GPL-2.0\n"+source, "Add GPL declaration") - out := cli(t, "log", repo, "--details") + out := cli(t, testLogCommand, repo, "--details") if !strings.Contains(out, "1 SPDX declarations added") || !strings.Contains(out, "SPDX declaration: false -> true") { t.Fatalf("a corpus-covered tag must count as a declaration:\n%s", out) } @@ -271,7 +277,7 @@ func TestCLISHA256(t *testing.T) { commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: Apache-2.0\n", "Change license") git(t, repo, "rm", "LICENSE") git(t, repo, "commit", "-m", "Remove license file") - out := cli(t, "log", repo) + out := cli(t, testLogCommand, repo) if !strings.Contains(out, "root: 3 commits; 1 expression changes; 1 file additions; 1 file deletions;") { t.Fatalf("SHA-256 history failed:\n%s", out) } @@ -281,7 +287,7 @@ func TestCLIMonthlyCounts(t *testing.T) { repo := repository(t) commitFile(t, repo, "source.c", mitNotice, "Add notice") commitFile(t, repo, "source.c", "// SPDX-License-Identifier: MIT\n"+mitNotice, "Add declaration") - out := cli(t, "log", repo, "--monthly") + out := cli(t, testLogCommand, repo, "--monthly") if !strings.HasPrefix(out, "month,group,commits,expression_changes,file_additions,file_deletions,incomplete,spdx_added,spdx_removed\n") || !strings.Contains(out, ",other,2,0,1,0,0,1,0\n") { t.Fatalf("monthly CSV must distinguish declaration additions:\n%s", out) } @@ -312,18 +318,18 @@ func TestIndexKeepsDeclarationWithoutDetection(t *testing.T) { func TestCLIEmptyRepository(t *testing.T) { repo := repository(t) - out := cli(t, "scan", repo) + out := cli(t, testScanCommand, repo) if metric(t, out, "blobs seen") != 0 { t.Fatalf("empty scan:\n%s", out) } - out = cli(t, "log", repo) + out = cli(t, testLogCommand, repo) if !strings.Contains(out, "root: 0 commits;") { t.Fatalf("empty history:\n%s", out) } } func TestCLIHelp(t *testing.T) { - out := cli(t, "log", "--help") + out := cli(t, testLogCommand, "--help") for _, option := range []string{"-max-legal-blob-size", "-group", "-monthly"} { if !strings.Contains(out, option) { t.Fatalf("help is missing %s:\n%s", option, out) @@ -332,7 +338,7 @@ func TestCLIHelp(t *testing.T) { if strings.Contains(out, "--cpuprofile") { t.Fatalf("log help includes scan-only flags:\n%s", out) } - scanHelp := cli(t, "scan", "--help") + scanHelp := cli(t, testScanCommand, "--help") if !strings.Contains(scanHelp, "--cpuprofile") || strings.Contains(scanHelp, "--monthly") { t.Fatalf("scan help has the wrong flags:\n%s", scanHelp) } @@ -341,8 +347,8 @@ func TestCLIHelp(t *testing.T) { func TestCLIFlagsMaySurroundRepository(t *testing.T) { repo := repository(t) commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: MIT\n", "Add license") - before := cli(t, "log", "--details", repo) - after := cli(t, "log", repo, "--details") + before := cli(t, testLogCommand, "--details", repo) + after := cli(t, testLogCommand, repo, "--details") if before != after { t.Fatalf("flag placement changed output:\nbefore repository:\n%s\nafter repository:\n%s", before, after) } @@ -351,10 +357,10 @@ func TestCLIFlagsMaySurroundRepository(t *testing.T) { func TestCLIExpressionSummaryBreaksCountTiesByName(t *testing.T) { repo := repository(t) ids := []string{ - "0BSD", "AGPL-3.0-only", "Apache-2.0", "Artistic-2.0", "BlueOak-1.0.0", + "0BSD", "AGPL-3.0-only", testApache, "Artistic-2.0", "BlueOak-1.0.0", "BSD-2-Clause", "BSD-3-Clause", "BSL-1.0", "CC0-1.0", "CDDL-1.0", "EPL-2.0", "EUPL-1.2", "GPL-2.0-only", "GPL-3.0-only", "ISC", - "LGPL-2.1-only", "MIT", "MIT-0", "MPL-2.0", "Unlicense", "WTFPL", "Zlib", + "LGPL-2.1-only", mitExpression, "MIT-0", "MPL-2.0", "Unlicense", "WTFPL", "Zlib", } for index, id := range ids { name := filepath.Join(repo, "file-"+strconv.Itoa(index)+".txt") @@ -365,7 +371,7 @@ func TestCLIExpressionSummaryBreaksCountTiesByName(t *testing.T) { git(t, repo, "add", ".") git(t, repo, "commit", "-m", "Add SPDX declarations") - out := cli(t, "scan", repo) + out := cli(t, testScanCommand, repo) summary := strings.Split(out, "expressions across all history:\n") if len(summary) != 2 { t.Fatal(out) @@ -395,7 +401,7 @@ func TestCLILegalPathIntroducedByMerge(t *testing.T) { commitFile(t, repo, "Godeps/LICENSES", content, "Add legal path during merge") git(t, repo, "rm", "Godeps/LICENSES") git(t, repo, "commit", "-m", "Remove legal path") - out := cli(t, "scan", repo, "--max-blob-size", "64", "--max-legal-blob-size", "1024") + out := cli(t, testScanCommand, repo, "--max-blob-size", "64", "--max-legal-blob-size", "1024") if metric(t, out, "blobs with hits") != 1 || metric(t, out, "skipped: size") != 0 { t.Fatalf("a merge-only legal path must contribute to blob eligibility:\n%s", out) } @@ -405,11 +411,11 @@ func TestCLIMatcherErrorRemainsIncomplete(t *testing.T) { repo := repository(t) commitFile(t, repo, "LICENSES/generated.txt", "SPDX-License-Identifier: MIT\n", "Add license") commitFile(t, repo, "LICENSES/generated.txt", strings.Repeat("mit license ", 400_000), "Exceed matcher candidate limit") - out := cli(t, "scan", repo) + out := cli(t, testScanCommand, repo) if metric(t, out, "skipped: error") != 1 { t.Fatalf("matcher errors must be counted:\n%s", out) } - out = cli(t, "log", repo, "--details") + out = cli(t, testLogCommand, repo, "--details") if !strings.Contains(out, "before=scanned, after=error") || !strings.Contains(out, "1 incomplete comparisons") || strings.Contains(out, " - MIT") { t.Fatalf("matcher errors must not imply license removal:\n%s", out) } @@ -470,14 +476,14 @@ func TestCLI(t *testing.T) { repo := repository(t) commitFile(t, repo, "source.go", "// SPDX-License-Identifier: MIT\npackage example\n", "Add source") commitFile(t, repo, "source.go", "// SPDX-License-Identifier: Apache-2.0\npackage example\n", "Change expression") - for _, command := range []string{"scan", "log"} { + for _, command := range []string{testScanCommand, testLogCommand} { t.Run(command, func(t *testing.T) { args := []string{command, repo, "--readers", "2"} - if command == "log" { + if command == testLogCommand { args = append(args, "--details") } out := cli(t, args...) - for _, expression := range []string{mitExpression, "Apache-2.0"} { + for _, expression := range []string{mitExpression, testApache} { if !strings.Contains(out, expression) { t.Fatalf("missing %s:\n%s", expression, out) } @@ -489,7 +495,7 @@ func TestCLI(t *testing.T) { func TestCLIBenchmarkPhases(t *testing.T) { repo := repository(t) commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: MIT\n", "Add license") - cmd := exec.Command(os.Args[0], "scan", repo, "--backend=gogit") + cmd := exec.Command(os.Args[0], testScanCommand, repo, "--backend=gogit") cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "GITSPDX_BENCH_PHASES=1", "GOMAXPROCS=2") out, err := cmd.CombinedOutput() if err != nil {