From d498aa065a7a26279c26dbf7ec6164cc0392fc3f Mon Sep 17 00:00:00 2001 From: Rex P Date: Wed, 7 Oct 2026 11:40:15 +1100 Subject: [PATCH 1/4] feat(ubuntu): scope binary-to-source mappings by Ubuntu release ecosystem --- AGENTS.md | 4 +- bindings/go/api/osv_service_v1.pb.go | 13 ++- docs/api/post-v1-ubuntu-binary-to-source.md | 11 +- docs/osv_service_v1.swagger.json | 3 + go/cmd/ubuntubinarymapper/main.go | 62 ++++++----- go/cmd/ubuntubinarymapper/main_test.go | 40 ++++--- go/internal/api/ubuntu_package_mapping.go | 24 ++++- .../api/ubuntu_package_mapping_test.go | 100 ++++++++++++++---- .../datastore/ubuntu_package_mapping.go | 42 ++++---- .../datastore/ubuntu_package_mapping_test.go | 34 ++++-- go/internal/database/jsonstore/jsonstore.go | 33 +++--- .../database/jsonstore/jsonstore_test.go | 31 ++++-- go/internal/models/ubuntu_package_mapping.go | 44 ++++++-- proto/v1/api_descriptor.pb | Bin 102056 -> 102145 bytes proto/v1/osv_service_v1.proto | 1 + 15 files changed, 320 insertions(+), 122 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 810ff965f3b..dad7d36da42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,8 +59,8 @@ These models are defined in Python ([`osv/models.py`](osv/models.py)) and mirror * **Rule**: This entity is **only** used by the website and should not be used for API matching logic. 5. **`UbuntuPackageMapping` (Kind: `UbuntuPackageMapping`)** - * **Purpose**: Used by the experimental `POST /v1experimental/ubuntu/binary-to-source` API endpoint to resolve Ubuntu binary package names to their corresponding source package names. - * **Fields**: Keyed by binary package name; stores `source_names` (unindexed list of corresponding Ubuntu source package names). + * **Purpose**: Used by the experimental `POST /v1experimental/ubuntu/binary-to-source` API endpoint to resolve Ubuntu binary package names to their corresponding source package names within an Ubuntu release ecosystem. + * **Fields**: Keyed by `:` (e.g., `Ubuntu:24.04:libcurl4`); stores `source_names` (unindexed list of corresponding Ubuntu source package names). * **Note**: Defined in Go ([`go/internal/database/datastore/ubuntu_package_mapping.go`](go/internal/database/datastore/ubuntu_package_mapping.go)) and populated by `ubuntubinarymapper`. --- diff --git a/bindings/go/api/osv_service_v1.pb.go b/bindings/go/api/osv_service_v1.pb.go index 6f703841b03..cae4d98f1c8 100644 --- a/bindings/go/api/osv_service_v1.pb.go +++ b/bindings/go/api/osv_service_v1.pb.go @@ -981,6 +981,7 @@ func (x *VersionRepositoryInformation) GetCommit() string { type UbuntuPackageMappingParameters struct { state protoimpl.MessageState `protogen:"open.v1"` BinaryNames []string `protobuf:"bytes,1,rep,name=binary_names,proto3" json:"binary_names,omitempty"` + Ecosystem string `protobuf:"bytes,2,opt,name=ecosystem,proto3" json:"ecosystem,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1022,6 +1023,13 @@ func (x *UbuntuPackageMappingParameters) GetBinaryNames() []string { return nil } +func (x *UbuntuPackageMappingParameters) GetEcosystem() string { + if x != nil { + return x.Ecosystem + } + return "" +} + // Source package names corresponding to a queried binary package name. type SourcePackages struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1171,9 +1179,10 @@ const file_osv_service_v1_proto_rawDesc = "" + "\x06commit\x18\x06 \x01(\tR\x06commit\"$\n" + "\bRepoType\x12\x0f\n" + "\vUNSPECIFIED\x10\x00\x12\a\n" + - "\x03GIT\x10\x01\"D\n" + + "\x03GIT\x10\x01\"b\n" + "\x1eUbuntuPackageMappingParameters\x12\"\n" + - "\fbinary_names\x18\x01 \x03(\tR\fbinary_names\"4\n" + + "\fbinary_names\x18\x01 \x03(\tR\fbinary_names\x12\x1c\n" + + "\tecosystem\x18\x02 \x01(\tR\tecosystem\"4\n" + "\x0eSourcePackages\x12\"\n" + "\fsource_names\x18\x01 \x03(\tR\fsource_names\"P\n" + "\x1cUbuntuPackageMappingResponse\x120\n" + diff --git a/docs/api/post-v1-ubuntu-binary-to-source.md b/docs/api/post-v1-ubuntu-binary-to-source.md index c4360ced10f..01ff04c25c1 100644 --- a/docs/api/post-v1-ubuntu-binary-to-source.md +++ b/docs/api/post-v1-ubuntu-binary-to-source.md @@ -40,14 +40,16 @@ package names. ## Parameters |--- -| Parameter | Type | Description | -| -------------- | ----- | ----------------------------------------------------------------------------- | -| `binary_names` | array | A non-empty array of Ubuntu binary package names (maximum 1000 per request). | +| Parameter | Type | Description | +| -------------- | ------ | -------------------------------------------------------------------------------------------------------------------- | +| `ecosystem` | string | The Ubuntu release ecosystem (e.g. `Ubuntu:24.04`, `Ubuntu:24.04:LTS`, or `Ubuntu:Pro:24.04:LTS`). | +| `binary_names` | array | A non-empty array of Ubuntu binary package names (maximum 1000 per request). | ## Payload ```json { + "ecosystem": "string", "binary_names": [ "string" ] @@ -59,6 +61,7 @@ package names. ```bash cat < slice of source_names from a Vulnerability record. -func ExtractBinaryMappings(vuln *osvschema.Vulnerability) map[string][]string { - mappings := make(map[string][]string) +// ExtractBinaryMappings extracts a map of (ecosystem, binary_name) -> slice of source_names from a Vulnerability record. +func ExtractBinaryMappings(vuln *osvschema.Vulnerability) map[models.UbuntuPackageKey][]string { + mappings := make(map[models.UbuntuPackageKey][]string) for _, affected := range vuln.GetAffected() { - if !strings.HasPrefix(affected.GetPackage().GetEcosystem(), "Ubuntu") { + rawEco := affected.GetPackage().GetEcosystem() + if !models.IsValidUbuntuReleaseEcosystem(rawEco) { continue } + eco := models.NormalizeUbuntuEcosystem(rawEco) sourceName := affected.GetPackage().GetName() if sourceName == "" { continue @@ -144,10 +146,17 @@ func ExtractBinaryMappings(vuln *osvschema.Vulnerability) map[string][]string { binaries := affected.GetEcosystemSpecific().GetFields()["binaries"].GetListValue().GetValues() for _, item := range binaries { binName := strings.TrimSpace(item.GetStructValue().GetFields()["binary_name"].GetStringValue()) + if binName == "" { + continue + } + key := models.UbuntuPackageKey{ + Ecosystem: eco, + BinaryName: binName, + } // A binary typically maps to only 1-2 unique source packages, so slices.Contains - // avoids duplicate appends across releases without per-binary map overhead. - if binName != "" && !slices.Contains(mappings[binName], sourceName) { - mappings[binName] = append(mappings[binName], sourceName) + // avoids duplicate appends without per-binary map overhead. + if !slices.Contains(mappings[key], sourceName) { + mappings[key] = append(mappings[key], sourceName) } } } @@ -260,7 +269,7 @@ func run(ctx context.Context, env *appEnv) error { if len(vulnIDs) > 0 { threshold := cmp.Or(env.zipThreshold, zipDownloadThreshold) workers := max(cmp.Or(env.numWorkers, defaultNumWorkers), 1) - var allMappings map[string][]string + var allMappings map[models.UbuntuPackageKey][]string if len(vulnIDs) > threshold { logger.InfoContext(ctx, "downloading Ubuntu/all.zip for bulk processing", slog.Int("count", len(vulnIDs)), slog.Int("threshold", threshold)) allMappings, err = extractMappingsFromAllZip(ctx, env.gcsStorage, vulnIDs, workers) @@ -272,7 +281,7 @@ func run(ctx context.Context, env *appEnv) error { return err } - logger.InfoContext(ctx, "extracted binary package mappings", slog.Int("unique_binaries", len(allMappings))) + logger.InfoContext(ctx, "extracted binary package mappings", slog.Int("unique_keys", len(allMappings))) if len(allMappings) > 0 { if err := saveMappings(ctx, env.ubuntuStore, allMappings); err != nil { return fmt.Errorf("failed saving mappings: %w", err) @@ -289,7 +298,7 @@ func run(ctx context.Context, env *appEnv) error { return nil } -func extractMappingsFromAllZip(ctx context.Context, gcsStorage clients.CloudStorage, vulnIDs []string, numWorkers int) (map[string][]string, error) { +func extractMappingsFromAllZip(ctx context.Context, gcsStorage clients.CloudStorage, vulnIDs []string, numWorkers int) (map[models.UbuntuPackageKey][]string, error) { zipBytes, err := gcsStorage.ReadObject(ctx, ubuntuAllZipPath) if err != nil { return nil, fmt.Errorf("failed reading %s: %w", ubuntuAllZipPath, err) @@ -306,7 +315,7 @@ func extractMappingsFromAllZip(ctx context.Context, gcsStorage clients.CloudStor } var mu sync.Mutex - allMappings := make(map[string][]string) + allMappings := make(map[models.UbuntuPackageKey][]string) g, gCtx := errgroup.WithContext(ctx) g.SetLimit(numWorkers) @@ -345,9 +354,9 @@ func extractMappingsFromAllZip(ctx context.Context, gcsStorage clients.CloudStor return allMappings, nil } -func extractMappingsFromIndividualFiles(ctx context.Context, gcsStorage clients.CloudStorage, vulnIDs []string, numWorkers int) (map[string][]string, error) { +func extractMappingsFromIndividualFiles(ctx context.Context, gcsStorage clients.CloudStorage, vulnIDs []string, numWorkers int) (map[models.UbuntuPackageKey][]string, error) { var mu sync.Mutex - allMappings := make(map[string][]string) + allMappings := make(map[models.UbuntuPackageKey][]string) g, ctx := errgroup.WithContext(ctx) g.SetLimit(numWorkers) @@ -377,7 +386,7 @@ func extractMappingsFromIndividualFiles(ctx context.Context, gcsStorage clients. return allMappings, nil } -func unmarshalAndMerge(data []byte, source string, mu *sync.Mutex, allMappings map[string][]string) error { +func unmarshalAndMerge(data []byte, source string, mu *sync.Mutex, allMappings map[models.UbuntuPackageKey][]string) error { var vuln osvschema.Vulnerability if err := (protojson.UnmarshalOptions{DiscardUnknown: true}).Unmarshal(data, &vuln); err != nil { return fmt.Errorf("failed unmarshaling %s: %w", source, err) @@ -390,12 +399,12 @@ func unmarshalAndMerge(data []byte, source string, mu *sync.Mutex, allMappings m mu.Lock() defer mu.Unlock() - for bin, sources := range recordMappings { + for key, sources := range recordMappings { for _, src := range sources { // A binary only has 1-2 source packages, so slices.Contains keeps the slice bounded // across thousands of records without allocating nested maps. - if !slices.Contains(allMappings[bin], src) { - allMappings[bin] = append(allMappings[bin], src) + if !slices.Contains(allMappings[key], src) { + allMappings[key] = append(allMappings[key], src) } } } @@ -403,26 +412,29 @@ func unmarshalAndMerge(data []byte, source string, mu *sync.Mutex, allMappings m return nil } -func saveMappings(ctx context.Context, store models.UbuntuPackageMappingStore, newMappings map[string][]string) error { - binaryNames := slices.Sorted(maps.Keys(newMappings)) +func saveMappings(ctx context.Context, store models.UbuntuPackageMappingStore, newMappings map[models.UbuntuPackageKey][]string) error { + keys := slices.SortedFunc(maps.Keys(newMappings), func(a, b models.UbuntuPackageKey) int { + return cmp.Or(cmp.Compare(a.Ecosystem, b.Ecosystem), cmp.Compare(a.BinaryName, b.BinaryName)) + }) - // GetMulti returns a 1:1 slice matching binaryNames. - existing, err := store.GetMulti(ctx, binaryNames) + // GetMulti returns a 1:1 slice matching keys. + existing, err := store.GetMulti(ctx, keys) if err != nil { return fmt.Errorf("failed getting existing mappings: %w", err) } - toPut := make([]*models.UbuntuPackageMapping, len(binaryNames)) - for i, bin := range binaryNames { + toPut := make([]*models.UbuntuPackageMapping, len(keys)) + for i, key := range keys { var existingSources []string if i < len(existing) && existing[i] != nil { existingSources = existing[i].SourceNames } - merged := slices.Concat(existingSources, newMappings[bin]) + merged := slices.Concat(existingSources, newMappings[key]) slices.Sort(merged) toPut[i] = &models.UbuntuPackageMapping{ - BinaryName: bin, + Ecosystem: key.Ecosystem, + BinaryName: key.BinaryName, SourceNames: slices.Compact(merged), } } diff --git a/go/cmd/ubuntubinarymapper/main_test.go b/go/cmd/ubuntubinarymapper/main_test.go index 9b50818fa7f..3601ffbbac5 100644 --- a/go/cmd/ubuntubinarymapper/main_test.go +++ b/go/cmd/ubuntubinarymapper/main_test.go @@ -60,7 +60,7 @@ func TestExtractBinaryMappings(t *testing.T) { { Package: &osvschema.Package{ Name: "glib2.0", - Ecosystem: "Ubuntu:24.04:LTS", + Ecosystem: "Ubuntu:Pro:24.04:LTS", }, EcosystemSpecific: ecoStruct, }, @@ -70,6 +70,13 @@ func TestExtractBinaryMappings(t *testing.T) { Ecosystem: "Ubuntu:22.04:LTS", }, }, + { + Package: &osvschema.Package{ + Name: "bare-ubuntu-pkg", + Ecosystem: "Ubuntu", + }, + EcosystemSpecific: ecoStruct, + }, { Package: &osvschema.Package{ Name: "debian-pkg", @@ -82,9 +89,11 @@ func TestExtractBinaryMappings(t *testing.T) { got := ExtractBinaryMappings(vuln) - expected := map[string][]string{ - "libglib2.0-0": {"glib2.0"}, - "libglib2.0-bin": {"glib2.0"}, + expected := map[models.UbuntuPackageKey][]string{ + {Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-0"}: {"glib2.0"}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-bin"}: {"glib2.0"}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "libglib2.0-0"}: {"glib2.0"}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "libglib2.0-bin"}: {"glib2.0"}, } if diff := cmp.Diff(expected, got); diff != "" { @@ -199,7 +208,7 @@ func TestRun_EndToEnd(t *testing.T) { Id: "USN-2-1", Affected: []*osvschema.Affected{ { - Package: &osvschema.Package{Name: "curl-esm-src", Ecosystem: "Ubuntu:22.04:LTS"}, + Package: &osvschema.Package{Name: "curl-esm-src", Ecosystem: "Ubuntu:Pro:22.04:LTS"}, EcosystemSpecific: ecoStruct2, }, }, @@ -238,13 +247,16 @@ func TestRun_EndToEnd(t *testing.T) { t.Fatalf("initial run failed: %v", err) } - got, err := ubuntuStore.GetMulti(ctx, []string{"curl", "libcurl4"}) + got, err := ubuntuStore.GetMulti(ctx, []models.UbuntuPackageKey{ + {Ecosystem: "Ubuntu:22.04", BinaryName: "curl"}, + {Ecosystem: "Ubuntu:22.04:LTS", BinaryName: "libcurl4"}, + }) if err != nil { t.Fatalf("GetMulti failed: %v", err) } wantInitial := []*models.UbuntuPackageMapping{ - {BinaryName: "curl", SourceNames: []string{"curl"}}, - {BinaryName: "libcurl4", SourceNames: []string{"curl"}}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "curl", SourceNames: []string{"curl"}}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "libcurl4", SourceNames: []string{"curl"}}, } if diff := cmp.Diff(wantInitial, got); diff != "" { t.Errorf("initial mappings mismatch (-want +got):\n%s", diff) @@ -262,14 +274,18 @@ func TestRun_EndToEnd(t *testing.T) { t.Fatalf("incremental run failed: %v", err) } - gotAfter, err := ubuntuStore.GetMulti(ctx, []string{"curl", "libcurl4", "curl-esm"}) + gotAfter, err := ubuntuStore.GetMulti(ctx, []models.UbuntuPackageKey{ + {Ecosystem: "Ubuntu:22.04", BinaryName: "curl"}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "libcurl4"}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "curl-esm"}, + }) if err != nil { t.Fatalf("GetMulti after incremental run failed: %v", err) } wantAfter := []*models.UbuntuPackageMapping{ - {BinaryName: "curl", SourceNames: []string{"curl"}}, - {BinaryName: "libcurl4", SourceNames: []string{"curl", "curl-esm-src"}}, - {BinaryName: "curl-esm", SourceNames: []string{"curl-esm-src"}}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "curl", SourceNames: []string{"curl"}}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "libcurl4", SourceNames: []string{"curl", "curl-esm-src"}}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "curl-esm", SourceNames: []string{"curl-esm-src"}}, } if diff := cmp.Diff(wantAfter, gotAfter); diff != "" { t.Errorf("mappings mismatch after incremental merge (-want +got):\n%s", diff) diff --git a/go/internal/api/ubuntu_package_mapping.go b/go/internal/api/ubuntu_package_mapping.go index ce9b61acc04..63b4115f4fa 100644 --- a/go/internal/api/ubuntu_package_mapping.go +++ b/go/internal/api/ubuntu_package_mapping.go @@ -18,6 +18,7 @@ import ( "context" "log/slog" + "github.com/google/osv.dev/go/internal/models" "github.com/google/osv.dev/go/logger" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -27,6 +28,7 @@ import ( const ( maxBinaryNames = 1000 maxBinaryNameLen = 256 + maxEcosystemLen = 256 ) // QueryUbuntuPackageMapping handles querying Ubuntu binary package names to retrieve their corresponding source package names. @@ -35,6 +37,15 @@ func (s *server) QueryUbuntuPackageMapping(ctx context.Context, params *pb.Ubunt return nil, status.Error(codes.Internal, "ubuntu package mapping store is not configured") } + rawEcosystem := params.GetEcosystem() + if rawEcosystem == "" { + return nil, status.Error(codes.InvalidArgument, "ecosystem is required") + } + if len(rawEcosystem) > maxEcosystemLen || !models.IsValidUbuntuReleaseEcosystem(rawEcosystem) { + return nil, status.Error(codes.InvalidArgument, "invalid ubuntu ecosystem: must include a release suffix (e.g. Ubuntu:22.04)") + } + ecosystem := models.NormalizeUbuntuEcosystem(rawEcosystem) + binaryNames := params.GetBinaryNames() if len(binaryNames) == 0 { return nil, status.Error(codes.InvalidArgument, "binary_names is required") @@ -43,18 +54,25 @@ func (s *server) QueryUbuntuPackageMapping(ctx context.Context, params *pb.Ubunt return nil, status.Errorf(codes.InvalidArgument, "too many binary_names (max %d)", maxBinaryNames) } + keys := make([]models.UbuntuPackageKey, len(binaryNames)) for i, name := range binaryNames { if name == "" || len(name) > maxBinaryNameLen { return nil, status.Errorf(codes.InvalidArgument, "invalid binary package name at index %d", i) } + keys[i] = models.UbuntuPackageKey{ + Ecosystem: ecosystem, + BinaryName: name, + } } if s.verboseLogs { - logger.InfoContext(ctx, "querying ubuntu package mapping", slog.Any("binary_names", binaryNames)) + logger.InfoContext(ctx, "querying ubuntu package mapping", + slog.String("ecosystem", ecosystem), + slog.Any("binary_names", binaryNames)) } - // GetMulti returns a 1:1 slice matching binaryNames. - mappings, err := s.ubuntuPackageMappingStore.GetMulti(ctx, binaryNames) + // GetMulti returns a 1:1 slice matching keys. + mappings, err := s.ubuntuPackageMappingStore.GetMulti(ctx, keys) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get ubuntu package mappings: %v", err) } diff --git a/go/internal/api/ubuntu_package_mapping_test.go b/go/internal/api/ubuntu_package_mapping_test.go index a361cbf6f65..1ff2b78fa07 100644 --- a/go/internal/api/ubuntu_package_mapping_test.go +++ b/go/internal/api/ubuntu_package_mapping_test.go @@ -31,23 +31,25 @@ import ( ) type mockUbuntuPackageMappingStore struct { - mappings map[string][]string + mappings map[models.UbuntuPackageKey][]string err error } -func (m *mockUbuntuPackageMappingStore) GetMulti(_ context.Context, binaryNames []string) ([]*models.UbuntuPackageMapping, error) { +func (m *mockUbuntuPackageMappingStore) GetMulti(_ context.Context, keys []models.UbuntuPackageKey) ([]*models.UbuntuPackageMapping, error) { if m.err != nil { return nil, m.err } - results := make([]*models.UbuntuPackageMapping, len(binaryNames)) - for i, name := range binaryNames { - sources, ok := m.mappings[name] + results := make([]*models.UbuntuPackageMapping, len(keys)) + for i, k := range keys { + eco := models.NormalizeUbuntuEcosystem(k.Ecosystem) + sources, ok := m.mappings[models.UbuntuPackageKey{Ecosystem: eco, BinaryName: k.BinaryName}] if !ok { sources = []string{} } results[i] = &models.UbuntuPackageMapping{ - BinaryName: name, + Ecosystem: eco, + BinaryName: k.BinaryName, SourceNames: sources, } } @@ -63,10 +65,10 @@ func TestQueryUbuntuPackageMapping(t *testing.T) { ctx := context.Background() mockStore := &mockUbuntuPackageMappingStore{ - mappings: map[string][]string{ - "libglib2.0-0": {"glib2.0"}, - "libglib2.0-bin": {"glib2.0"}, - "shared-bin": {"src-1", "src-2"}, + mappings: map[models.UbuntuPackageKey][]string{ + {Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-0"}: {"glib2.0"}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-bin"}: {"glib2.0"}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "shared-bin"}: {"src-1", "src-2"}, }, } @@ -90,6 +92,7 @@ func TestQueryUbuntuPackageMapping(t *testing.T) { { name: "Success with existing, duplicate, and non-existing binaries", params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:22.04:LTS", BinaryNames: []string{"libglib2.0-0", "unknown-binary", "shared-bin", "libglib2.0-0"}, }, wantResp: &pb.UbuntuPackageMappingResponse{ @@ -111,8 +114,59 @@ func TestQueryUbuntuPackageMapping(t *testing.T) { wantCode: codes.OK, }, { - name: "Empty binary names", - params: &pb.UbuntuPackageMappingParameters{BinaryNames: []string{}}, + name: "Success with normalized Pro and bare release ecosystem", + params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:Pro:22.04:LTS", + BinaryNames: []string{"libglib2.0-0"}, + }, + wantResp: &pb.UbuntuPackageMappingResponse{ + Results: []*pb.SourcePackages{ + { + SourceNames: []string{"glib2.0"}, + }, + }, + }, + wantCode: codes.OK, + }, + { + name: "Missing ecosystem", + params: &pb.UbuntuPackageMappingParameters{BinaryNames: []string{"libglib2.0-0"}}, + wantCode: codes.InvalidArgument, + wantError: "ecosystem is required", + }, + { + name: "Bare Ubuntu ecosystem without release suffix", + params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu", + BinaryNames: []string{"libglib2.0-0"}, + }, + wantCode: codes.InvalidArgument, + wantError: "invalid ubuntu ecosystem", + }, + { + name: "Ubuntu ecosystem with only LTS or Pro modifier and no version", + params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:Pro:LTS", + BinaryNames: []string{"libglib2.0-0"}, + }, + wantCode: codes.InvalidArgument, + wantError: "invalid ubuntu ecosystem", + }, + { + name: "Non-Ubuntu ecosystem", + params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Debian:12", + BinaryNames: []string{"libglib2.0-0"}, + }, + wantCode: codes.InvalidArgument, + wantError: "invalid ubuntu ecosystem", + }, + { + name: "Empty binary names", + params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:22.04", + BinaryNames: []string{}, + }, wantCode: codes.InvalidArgument, wantError: "binary_names is required", }, @@ -120,23 +174,30 @@ func TestQueryUbuntuPackageMapping(t *testing.T) { name: "Nil parameters", params: nil, wantCode: codes.InvalidArgument, - wantError: "binary_names is required", + wantError: "ecosystem is required", }, { - name: "Empty string in binary names", - params: &pb.UbuntuPackageMappingParameters{BinaryNames: []string{"libglib2.0-0", ""}}, + name: "Empty string in binary names", + params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:22.04", + BinaryNames: []string{"libglib2.0-0", ""}, + }, wantCode: codes.InvalidArgument, wantError: "invalid binary package name at index 1", }, { - name: "Too many binary names", - params: &pb.UbuntuPackageMappingParameters{BinaryNames: tooManyNames}, + name: "Too many binary names", + params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:22.04", + BinaryNames: tooManyNames, + }, wantCode: codes.InvalidArgument, wantError: "too many binary_names", }, { name: "Store failure propagates as internal error", params: &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:22.04", BinaryNames: []string{"libglib2.0-0"}, }, storeErr: errors.New("connection failed"), @@ -186,8 +247,8 @@ func TestQueryUbuntuPackageMapping_JSONStore(t *testing.T) { } if err := store.PutMulti(ctx, []*models.UbuntuPackageMapping{ - {BinaryName: "libglib2.0-0", SourceNames: []string{"glib2.0"}}, - {BinaryName: "libcurl4", SourceNames: []string{"curl"}}, + {Ecosystem: "Ubuntu:24.04:LTS", BinaryName: "libglib2.0-0", SourceNames: []string{"glib2.0"}}, + {Ecosystem: "Ubuntu:24.04:LTS", BinaryName: "libcurl4", SourceNames: []string{"curl"}}, }); err != nil { t.Fatalf("PutMulti failed: %v", err) } @@ -197,6 +258,7 @@ func TestQueryUbuntuPackageMapping_JSONStore(t *testing.T) { } resp, err := srv.QueryUbuntuPackageMapping(ctx, &pb.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:24.04", BinaryNames: []string{"libglib2.0-0", "libcurl4", "nonexistent"}, }) if err != nil { diff --git a/go/internal/database/datastore/ubuntu_package_mapping.go b/go/internal/database/datastore/ubuntu_package_mapping.go index a08e5600bd4..4b85be91df9 100644 --- a/go/internal/database/datastore/ubuntu_package_mapping.go +++ b/go/internal/database/datastore/ubuntu_package_mapping.go @@ -47,19 +47,24 @@ func NewUbuntuPackageMappingStore(client *datastore.Client) *UbuntuPackageMappin return &UbuntuPackageMappingStore{client: client} } -// GetMulti retrieves mappings for a slice of binary package names. -// Any binary not found in Datastore will return an UbuntuPackageMapping with an empty SourceNames slice. -func (s *UbuntuPackageMappingStore) GetMulti(ctx context.Context, binaryNames []string) ([]*models.UbuntuPackageMapping, error) { - results := make([]*models.UbuntuPackageMapping, 0, len(binaryNames)) - - for chunkNames := range slices.Chunk(binaryNames, maxDatastoreGetMultiBatchSize) { - keys := make([]*datastore.Key, len(chunkNames)) - entities := make([]*ubuntuPackageMappingEntity, len(chunkNames)) - for j, name := range chunkNames { - keys[j] = datastore.NameKey(UbuntuPackageMappingKind, name, nil) +func makeDatastoreKey(ecosystem, binaryName string) *datastore.Key { + normalizedEco := models.NormalizeUbuntuEcosystem(ecosystem) + return datastore.NameKey(UbuntuPackageMappingKind, normalizedEco+":"+binaryName, nil) +} + +// GetMulti retrieves mappings for a slice of (ecosystem, binary_name) keys. +// Any key not found in Datastore will return an UbuntuPackageMapping with an empty SourceNames slice. +func (s *UbuntuPackageMappingStore) GetMulti(ctx context.Context, pkgKeys []models.UbuntuPackageKey) ([]*models.UbuntuPackageMapping, error) { + results := make([]*models.UbuntuPackageMapping, 0, len(pkgKeys)) + + for chunkKeys := range slices.Chunk(pkgKeys, maxDatastoreGetMultiBatchSize) { + dsKeys := make([]*datastore.Key, len(chunkKeys)) + entities := make([]*ubuntuPackageMappingEntity, len(chunkKeys)) + for j, k := range chunkKeys { + dsKeys[j] = makeDatastoreKey(k.Ecosystem, k.BinaryName) } - if err := s.client.GetMulti(ctx, keys, entities); err != nil { + if err := s.client.GetMulti(ctx, dsKeys, entities); err != nil { multiErr, ok := errors.AsType[datastore.MultiError](err) if !ok { return nil, fmt.Errorf("failed to get multi UbuntuPackageMapping: %w", err) @@ -73,13 +78,14 @@ func (s *UbuntuPackageMappingStore) GetMulti(ctx context.Context, binaryNames [] } } - for j, name := range chunkNames { + for j, k := range chunkKeys { sourceNames := []string{} if entities[j] != nil && len(entities[j].SourceNames) > 0 { sourceNames = entities[j].SourceNames } results = append(results, &models.UbuntuPackageMapping{ - BinaryName: name, + Ecosystem: models.NormalizeUbuntuEcosystem(k.Ecosystem), + BinaryName: k.BinaryName, SourceNames: sourceNames, }) } @@ -88,25 +94,25 @@ func (s *UbuntuPackageMappingStore) GetMulti(ctx context.Context, binaryNames [] return results, nil } -// PutMulti creates or updates package mappings for multiple binary package names. +// PutMulti creates or updates package mappings for multiple (ecosystem, binary_name) pairs. func (s *UbuntuPackageMappingStore) PutMulti(ctx context.Context, mappings []*models.UbuntuPackageMapping) error { for chunk := range slices.Chunk(mappings, maxDatastorePutMultiBatchSize) { - keys := make([]*datastore.Key, 0, len(chunk)) + dsKeys := make([]*datastore.Key, 0, len(chunk)) entities := make([]*ubuntuPackageMappingEntity, 0, len(chunk)) for _, m := range chunk { if m == nil { continue } - keys = append(keys, datastore.NameKey(UbuntuPackageMappingKind, m.BinaryName, nil)) + dsKeys = append(dsKeys, makeDatastoreKey(m.Ecosystem, m.BinaryName)) entities = append(entities, &ubuntuPackageMappingEntity{ SourceNames: m.SourceNames, }) } - if len(keys) == 0 { + if len(dsKeys) == 0 { continue } - if _, err := s.client.PutMulti(ctx, keys, entities); err != nil { + if _, err := s.client.PutMulti(ctx, dsKeys, entities); err != nil { return fmt.Errorf("failed to put multi UbuntuPackageMapping: %w", err) } } diff --git a/go/internal/database/datastore/ubuntu_package_mapping_test.go b/go/internal/database/datastore/ubuntu_package_mapping_test.go index 964dbc58f01..29915e535ce 100644 --- a/go/internal/database/datastore/ubuntu_package_mapping_test.go +++ b/go/internal/database/datastore/ubuntu_package_mapping_test.go @@ -46,14 +46,17 @@ func TestUbuntuPackageMappingStore_GetAndPutMulti(t *testing.T) { // Insert initial mappings initialMappings := []*models.UbuntuPackageMapping{ { + Ecosystem: "Ubuntu:22.04:LTS", BinaryName: "libglib2.0-0", SourceNames: []string{"glib2.0"}, }, { + Ecosystem: "Ubuntu:24.04", BinaryName: "libglib2.0-bin", SourceNames: []string{"glib2.0"}, }, { + Ecosystem: "Ubuntu:22.04", BinaryName: "shared-bin", SourceNames: []string{"src-a", "src-b"}, }, @@ -63,22 +66,29 @@ func TestUbuntuPackageMappingStore_GetAndPutMulti(t *testing.T) { t.Fatalf("PutMulti failed: %v", err) } - // Query existing and non-existing binaries - queried, err := store.GetMulti(ctx, []string{"libglib2.0-0", "nonexistent-pkg", "shared-bin"}) + // Query existing and non-existing binaries across normalized variants + queried, err := store.GetMulti(ctx, []models.UbuntuPackageKey{ + {Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-0"}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "nonexistent-pkg"}, + {Ecosystem: "Ubuntu:Pro:22.04:LTS", BinaryName: "shared-bin"}, + }) if err != nil { t.Fatalf("GetMulti failed: %v", err) } expected := []*models.UbuntuPackageMapping{ { + Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-0", SourceNames: []string{"glib2.0"}, }, { + Ecosystem: "Ubuntu:22.04", BinaryName: "nonexistent-pkg", SourceNames: []string{}, }, { + Ecosystem: "Ubuntu:22.04", BinaryName: "shared-bin", SourceNames: []string{"src-a", "src-b"}, }, @@ -91,6 +101,7 @@ func TestUbuntuPackageMappingStore_GetAndPutMulti(t *testing.T) { // Test update updateMappings := []*models.UbuntuPackageMapping{ { + Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-0", SourceNames: []string{"glib2.0", "glib2.0-esm"}, }, @@ -99,12 +110,15 @@ func TestUbuntuPackageMappingStore_GetAndPutMulti(t *testing.T) { t.Fatalf("PutMulti update failed: %v", err) } - updated, err := store.GetMulti(ctx, []string{"libglib2.0-0"}) + updated, err := store.GetMulti(ctx, []models.UbuntuPackageKey{ + {Ecosystem: "Ubuntu:22.04:LTS", BinaryName: "libglib2.0-0"}, + }) if err != nil { t.Fatalf("GetMulti after update failed: %v", err) } expectedUpdate := []*models.UbuntuPackageMapping{ { + Ecosystem: "Ubuntu:22.04", BinaryName: "libglib2.0-0", SourceNames: []string{"glib2.0", "glib2.0-esm"}, }, @@ -122,11 +136,15 @@ func TestUbuntuPackageMappingStore_BatchChunking(t *testing.T) { // Create 550 entries to verify chunking (> 500 maxDatastorePutMultiBatchSize) total := 550 mappings := make([]*models.UbuntuPackageMapping, total) - names := make([]string, total) + keys := make([]models.UbuntuPackageKey, total) for i := range total { name := fmt.Sprintf("pkg-batch-%d", i) - names[i] = name + keys[i] = models.UbuntuPackageKey{ + Ecosystem: "Ubuntu:24.04", + BinaryName: name, + } mappings[i] = &models.UbuntuPackageMapping{ + Ecosystem: "Ubuntu:24.04", BinaryName: name, SourceNames: []string{fmt.Sprintf("src-%d", i)}, } @@ -136,7 +154,7 @@ func TestUbuntuPackageMappingStore_BatchChunking(t *testing.T) { t.Fatalf("PutMulti chunking failed: %v", err) } - queried, err := store.GetMulti(ctx, names) + queried, err := store.GetMulti(ctx, keys) if err != nil { t.Fatalf("GetMulti chunking failed: %v", err) } @@ -145,8 +163,8 @@ func TestUbuntuPackageMappingStore_BatchChunking(t *testing.T) { t.Fatalf("expected %d queried mappings, got %d", total, len(queried)) } for i := range total { - if queried[i].BinaryName != names[i] { - t.Errorf("index %d: expected name %q, got %q", i, names[i], queried[i].BinaryName) + if queried[i].BinaryName != keys[i].BinaryName { + t.Errorf("index %d: expected name %q, got %q", i, keys[i].BinaryName, queried[i].BinaryName) } if len(queried[i].SourceNames) != 1 || queried[i].SourceNames[0] != fmt.Sprintf("src-%d", i) { t.Errorf("index %d: unexpected source names: %v", i, queried[i].SourceNames) diff --git a/go/internal/database/jsonstore/jsonstore.go b/go/internal/database/jsonstore/jsonstore.go index d266ad31a7e..063da4ef66b 100644 --- a/go/internal/database/jsonstore/jsonstore.go +++ b/go/internal/database/jsonstore/jsonstore.go @@ -33,7 +33,7 @@ import ( type JSONStore struct { mu sync.RWMutex filePath string - mappings map[string][]string + mappings map[string]map[string][]string } var _ models.UbuntuPackageMappingStore = (*JSONStore)(nil) @@ -45,7 +45,7 @@ var _ models.UbuntuPackageMappingStore = (*JSONStore)(nil) func New(filePath string) (*JSONStore, error) { store := &JSONStore{ filePath: filePath, - mappings: make(map[string][]string), + mappings: make(map[string]map[string][]string), } if filePath == "" { @@ -65,7 +65,7 @@ func New(filePath string) (*JSONStore, error) { return store, nil } - var mappings map[string][]string + var mappings map[string]map[string][]string if err := json.Unmarshal(dataBytes, &mappings); err != nil { return nil, fmt.Errorf("failed unmarshaling JSON store file %s: %w", filePath, err) } @@ -77,19 +77,24 @@ func New(filePath string) (*JSONStore, error) { return store, nil } -// GetMulti retrieves package mappings for multiple binary names. -func (s *JSONStore) GetMulti(_ context.Context, binaryNames []string) ([]*models.UbuntuPackageMapping, error) { +// GetMulti retrieves package mappings for multiple (ecosystem, binary_name) keys. +func (s *JSONStore) GetMulti(_ context.Context, keys []models.UbuntuPackageKey) ([]*models.UbuntuPackageMapping, error) { s.mu.RLock() defer s.mu.RUnlock() - results := make([]*models.UbuntuPackageMapping, len(binaryNames)) - for i, name := range binaryNames { - srcs, ok := s.mappings[name] - if !ok || srcs == nil { + results := make([]*models.UbuntuPackageMapping, len(keys)) + for i, k := range keys { + eco := models.NormalizeUbuntuEcosystem(k.Ecosystem) + var srcs []string + if ecoMap, ok := s.mappings[eco]; ok { + srcs = ecoMap[k.BinaryName] + } + if srcs == nil { srcs = []string{} } results[i] = &models.UbuntuPackageMapping{ - BinaryName: name, + Ecosystem: eco, + BinaryName: k.BinaryName, SourceNames: slices.Clone(srcs), } } @@ -97,7 +102,7 @@ func (s *JSONStore) GetMulti(_ context.Context, binaryNames []string) ([]*models return results, nil } -// PutMulti stores or updates mappings for multiple binary package names. +// PutMulti stores or updates mappings for multiple (ecosystem, binary_name) pairs. func (s *JSONStore) PutMulti(_ context.Context, mappings []*models.UbuntuPackageMapping) error { s.mu.Lock() defer s.mu.Unlock() @@ -106,9 +111,13 @@ func (s *JSONStore) PutMulti(_ context.Context, mappings []*models.UbuntuPackage if item == nil { continue } + eco := models.NormalizeUbuntuEcosystem(item.Ecosystem) + if s.mappings[eco] == nil { + s.mappings[eco] = make(map[string][]string) + } srcs := slices.Clone(item.SourceNames) slices.Sort(srcs) - s.mappings[item.BinaryName] = slices.Compact(srcs) + s.mappings[eco][item.BinaryName] = slices.Compact(srcs) } if s.filePath == "" { diff --git a/go/internal/database/jsonstore/jsonstore_test.go b/go/internal/database/jsonstore/jsonstore_test.go index 9ceab240802..7c064b7677d 100644 --- a/go/internal/database/jsonstore/jsonstore_test.go +++ b/go/internal/database/jsonstore/jsonstore_test.go @@ -31,7 +31,10 @@ func TestJSONStore_InMemory(t *testing.T) { } // Initial GetMulti should return empty SourceNames - initial, err := store.GetMulti(ctx, []string{"pkg-a", "pkg-b"}) + initial, err := store.GetMulti(ctx, []models.UbuntuPackageKey{ + {Ecosystem: "Ubuntu:24.04", BinaryName: "pkg-a"}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "pkg-b"}, + }) if err != nil { t.Fatalf("GetMulti failed: %v", err) } @@ -44,23 +47,27 @@ func TestJSONStore_InMemory(t *testing.T) { // Put mappings err = store.PutMulti(ctx, []*models.UbuntuPackageMapping{ - {BinaryName: "pkg-a", SourceNames: []string{"src-1"}}, - {BinaryName: "pkg-b", SourceNames: []string{"src-2", "src-1"}}, + {Ecosystem: "Ubuntu:24.04:LTS", BinaryName: "pkg-a", SourceNames: []string{"src-1"}}, + {Ecosystem: "Ubuntu:Pro:22.04:LTS", BinaryName: "pkg-b", SourceNames: []string{"src-2", "src-1"}}, }) if err != nil { t.Fatalf("PutMulti failed: %v", err) } - // Verify GetMulti returns sorted, deduplicated sources - queried, err := store.GetMulti(ctx, []string{"pkg-b", "pkg-a", "unknown"}) + // Verify GetMulti returns sorted, deduplicated sources with normalized ecosystems + queried, err := store.GetMulti(ctx, []models.UbuntuPackageKey{ + {Ecosystem: "Ubuntu:22.04", BinaryName: "pkg-b"}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "pkg-a"}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "unknown"}, + }) if err != nil { t.Fatalf("GetMulti failed: %v", err) } expected := []*models.UbuntuPackageMapping{ - {BinaryName: "pkg-b", SourceNames: []string{"src-1", "src-2"}}, - {BinaryName: "pkg-a", SourceNames: []string{"src-1"}}, - {BinaryName: "unknown", SourceNames: []string{}}, + {Ecosystem: "Ubuntu:22.04", BinaryName: "pkg-b", SourceNames: []string{"src-1", "src-2"}}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "pkg-a", SourceNames: []string{"src-1"}}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "unknown", SourceNames: []string{}}, } if diff := cmp.Diff(expected, queried); diff != "" { t.Errorf("GetMulti mismatch (-want +got):\n%s", diff) @@ -78,7 +85,7 @@ func TestJSONStore_FilePersistence(t *testing.T) { } if err := store1.PutMulti(ctx, []*models.UbuntuPackageMapping{ - {BinaryName: "libcurl4", SourceNames: []string{"curl"}}, + {Ecosystem: "Ubuntu:24.04:LTS", BinaryName: "libcurl4", SourceNames: []string{"curl"}}, }); err != nil { t.Fatalf("PutMulti failed: %v", err) } @@ -89,12 +96,14 @@ func TestJSONStore_FilePersistence(t *testing.T) { t.Fatalf("New on existing file failed: %v", err) } - queried, err := store2.GetMulti(ctx, []string{"libcurl4"}) + queried, err := store2.GetMulti(ctx, []models.UbuntuPackageKey{ + {Ecosystem: "Ubuntu:24.04", BinaryName: "libcurl4"}, + }) if err != nil { t.Fatalf("store2 GetMulti failed: %v", err) } expected := []*models.UbuntuPackageMapping{ - {BinaryName: "libcurl4", SourceNames: []string{"curl"}}, + {Ecosystem: "Ubuntu:24.04", BinaryName: "libcurl4", SourceNames: []string{"curl"}}, } if diff := cmp.Diff(expected, queried); diff != "" { t.Errorf("store2 mismatch (-want +got):\n%s", diff) diff --git a/go/internal/models/ubuntu_package_mapping.go b/go/internal/models/ubuntu_package_mapping.go index 79c10c06032..f980c127a87 100644 --- a/go/internal/models/ubuntu_package_mapping.go +++ b/go/internal/models/ubuntu_package_mapping.go @@ -16,11 +16,43 @@ package models import ( "context" + "strings" ) -// UbuntuPackageMapping represents the mapping from an Ubuntu binary package name +var ubuntuVariantReplacer = strings.NewReplacer(":Pro", "", ":LTS", "") + +// NormalizeUbuntuEcosystem normalizes an Ubuntu ecosystem string by removing +// optional ":Pro" and ":LTS" modifiers (e.g. "Ubuntu:24.04:LTS" -> "Ubuntu:24.04"). +func NormalizeUbuntuEcosystem(ecosystem string) string { + if strings.HasPrefix(ecosystem, "Ubuntu") { + return ubuntuVariantReplacer.Replace(ecosystem) + } + + return ecosystem +} + +// IsValidUbuntuReleaseEcosystem reports whether an ecosystem string (after normalization) +// represents a valid Ubuntu release ecosystem with a non-empty release suffix (e.g. "Ubuntu:22.04"). +func IsValidUbuntuReleaseEcosystem(ecosystem string) bool { + normalized := NormalizeUbuntuEcosystem(ecosystem) + suffix, ok := strings.CutPrefix(normalized, "Ubuntu:") + if !ok || suffix == "" { + return false + } + + return !strings.Contains(suffix, "::") && !strings.HasPrefix(suffix, ":") && !strings.HasSuffix(suffix, ":") +} + +// UbuntuPackageKey uniquely identifies an Ubuntu binary package within a specific Ubuntu release ecosystem. +type UbuntuPackageKey struct { + Ecosystem string + BinaryName string +} + +// UbuntuPackageMapping represents the mapping from an Ubuntu ecosystem and binary package name // to its corresponding source package names. type UbuntuPackageMapping struct { + Ecosystem string BinaryName string SourceNames []string } @@ -28,11 +60,11 @@ type UbuntuPackageMapping struct { // UbuntuPackageMappingStore abstracts database operations for querying and updating // Ubuntu binary-to-source package name mappings. type UbuntuPackageMappingStore interface { - // GetMulti retrieves mappings for a slice of binary package names. - // For any binary package not found, the returned UbuntuPackageMapping has an empty SourceNames slice. - // This results in a 1:1 mapping of input binaryNames to the returned SourceNames. - GetMulti(ctx context.Context, binaryNames []string) ([]*UbuntuPackageMapping, error) + // GetMulti retrieves mappings for a slice of (ecosystem, binary_name) keys. + // For any key not found, the returned UbuntuPackageMapping has an empty SourceNames slice. + // This results in a 1:1 mapping of input keys to the returned UbuntuPackageMapping slice. + GetMulti(ctx context.Context, keys []UbuntuPackageKey) ([]*UbuntuPackageMapping, error) - // PutMulti creates or updates package mappings for multiple binary package names. + // PutMulti creates or updates package mappings for multiple (ecosystem, binary_name) pairs. PutMulti(ctx context.Context, mappings []*UbuntuPackageMapping) error } diff --git a/proto/v1/api_descriptor.pb b/proto/v1/api_descriptor.pb index 9d8967ae432d86c5df3717d4b5ddbbb96a2a3600..3cbe335bb36e9e2adb11a1c8652f7847f940973f 100644 GIT binary patch delta 978 zcmY+?-%FEG9LMqJbDr~@wqvC`{+j(*YO{hUQpviQ-Dri-MOZ8dQZT%dSmwOws=PDJ ztCGhRTOfp zlRsmJ%f(rYD=hmh^C({0D>-juQ2Z%ayWA(epgf%+oBA}YY`Vc|Yn!|-8E;Zz1w{@Hu+zf}c?J7|L``~|n zjG<%&qRn6~*%h?++9bPz#1&(^WWNJ{U$7^>zCx3PcXOdNj7K3rJt7tuC zlk6(aj~JWNwU6Lib(5<| VFW}$mwrA|!;e@z*FO^R9{{;i4nacnG delta 897 zcmY+?-%C?r9LMqVdCqgbt!I_)acwGOX|srGOF%?$Iu;F_3!O&Q`?@D3MW!`Zopll+VhgiraJG6sjkfAlEXe#Al{olCFEx-px~SNbD?Dg`nYfT$yNoKoa*VM)btke z9Z2=5z5a8#(fT^l9nJ?fG*uxs;0|6f=_Mr1%46iNRY84L1uv;0elkp6QpMyYy9B)C zBjl&<64)hnSVyo$XsSVcF(;O=Rj7gbttJ?+VPGOm#<$QjYH`7M9rAM*7Z$g}d|V)1}I@Mm+c0UV_~x0K=!h=UADhLe(&0@we4^$u=ztxh?d!2#+gDB6t|dU zyNQFR!(_XOb5quq{Mt{*UtL>T+YaXf+tbvB*fBFtaDBQBI&3kI?KX}Ce{TNOJht1I ax@m2X?QO{4UE8yEXLy5d6&LR;j{gG;hl*7I diff --git a/proto/v1/osv_service_v1.proto b/proto/v1/osv_service_v1.proto index 1cf667fd6df..9ba70264103 100644 --- a/proto/v1/osv_service_v1.proto +++ b/proto/v1/osv_service_v1.proto @@ -146,6 +146,7 @@ message VersionRepositoryInformation { // Parameters for QueryUbuntuPackageMapping. message UbuntuPackageMappingParameters { repeated string binary_names = 1 [json_name="binary_names"]; + string ecosystem = 2; } // Source package names corresponding to a queried binary package name. From a42ae480380ad25f8a9cadf8b48a6cf062deaaf5 Mon Sep 17 00:00:00 2001 From: Rex P Date: Wed, 7 Oct 2026 13:10:28 +1100 Subject: [PATCH 2/4] refactor(ubuntu): consolidate NormalizeUbuntuEcosystem and tighten validation --- .../database/datastore/affected_versions.go | 3 +- .../datastore/listed_vulnerability.go | 14 +--- go/internal/models/ubuntu_package_mapping.go | 2 +- .../models/ubuntu_package_mapping_test.go | 73 +++++++++++++++++++ 4 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 go/internal/models/ubuntu_package_mapping_test.go diff --git a/go/internal/database/datastore/affected_versions.go b/go/internal/database/datastore/affected_versions.go index cc372c82fc5..ac693ecf497 100644 --- a/go/internal/database/datastore/affected_versions.go +++ b/go/internal/database/datastore/affected_versions.go @@ -5,6 +5,7 @@ import ( "slices" "strings" + "github.com/google/osv.dev/go/internal/models" "github.com/google/osv.dev/go/internal/osvutil" "github.com/google/osv.dev/go/osv/ecosystem" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -45,7 +46,7 @@ func computeAffectedVersions(vuln *osvschema.Vulnerability) []AffectedVersions { if normalized != pkgEcosystem { allPkgEcosystems = append(allPkgEcosystems, normalized) } - if v := removeVariants(pkgEcosystem); v != "" { + if v := models.NormalizeUbuntuEcosystem(pkgEcosystem); v != pkgEcosystem { allPkgEcosystems = append(allPkgEcosystems, v) } diff --git a/go/internal/database/datastore/listed_vulnerability.go b/go/internal/database/datastore/listed_vulnerability.go index 82e95dc34c5..4c68e7c47fd 100644 --- a/go/internal/database/datastore/listed_vulnerability.go +++ b/go/internal/database/datastore/listed_vulnerability.go @@ -21,6 +21,7 @@ import ( "unicode" "unicode/utf8" + "github.com/google/osv.dev/go/internal/models" "github.com/ossf/osv-schema/bindings/go/osvschema" ) @@ -63,17 +64,6 @@ func tokenize(value string) []string { return result } -var ubuntuReplacer = strings.NewReplacer(":Pro", "", ":LTS", "") - -func removeVariants(ecosystem string) string { - // For Ubuntu, remove ":Pro" and ":LTS" - if strings.HasPrefix(ecosystem, "Ubuntu") { - return ubuntuReplacer.Replace(ecosystem) - } - - return "" -} - // truncate ensures that the given string is shorter than 80 characters. // // If the string is longer than that limit, it's trimmed and suffixed with an ellipsis. @@ -224,7 +214,7 @@ func NewListedVulnerabilityFromProto(vuln *osvschema.Vulnerability) *ListedVulne } searchIndices[t] = struct{}{} } - if v := removeVariants(eco); v != "" { + if v := models.NormalizeUbuntuEcosystem(eco); v != eco { for _, t := range tokenize(v) { if len(searchIndices) >= maxIndices { break diff --git a/go/internal/models/ubuntu_package_mapping.go b/go/internal/models/ubuntu_package_mapping.go index f980c127a87..3d1ddf4aafc 100644 --- a/go/internal/models/ubuntu_package_mapping.go +++ b/go/internal/models/ubuntu_package_mapping.go @@ -36,7 +36,7 @@ func NormalizeUbuntuEcosystem(ecosystem string) string { func IsValidUbuntuReleaseEcosystem(ecosystem string) bool { normalized := NormalizeUbuntuEcosystem(ecosystem) suffix, ok := strings.CutPrefix(normalized, "Ubuntu:") - if !ok || suffix == "" { + if !ok || suffix == "" || strings.ContainsAny(suffix, " \t\n\r") { return false } diff --git a/go/internal/models/ubuntu_package_mapping_test.go b/go/internal/models/ubuntu_package_mapping_test.go new file mode 100644 index 00000000000..07eb79b3a0a --- /dev/null +++ b/go/internal/models/ubuntu_package_mapping_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import "testing" + +func TestNormalizeUbuntuEcosystem(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want string + }{ + {"Ubuntu:22.04", "Ubuntu:22.04"}, + {"Ubuntu:22.04:LTS", "Ubuntu:22.04"}, + {"Ubuntu:Pro:22.04:LTS", "Ubuntu:22.04"}, + {"Ubuntu:Pro:FIPS-updates:22.04:LTS", "Ubuntu:FIPS-updates:22.04"}, + {"Ubuntu:25.04", "Ubuntu:25.04"}, + {"Ubuntu", "Ubuntu"}, + {"Debian:12", "Debian:12"}, + } + + for _, tc := range tests { + if got := NormalizeUbuntuEcosystem(tc.input); got != tc.want { + t.Errorf("NormalizeUbuntuEcosystem(%q) = %q, want %q", tc.input, got, tc.want) + } + } +} + +func TestIsValidUbuntuReleaseEcosystem(t *testing.T) { + t.Parallel() + + tests := []struct { + input string + want bool + }{ + {"Ubuntu:22.04", true}, + {"Ubuntu:22.04:LTS", true}, + {"Ubuntu:Pro:18.04:LTS", true}, + {"Ubuntu:Pro:FIPS-updates:22.04:LTS", true}, + {"Ubuntu:Nvidia-BlueField:22.04:LTS", true}, + {"Ubuntu:25.04", true}, + {"", false}, + {"Ubuntu", false}, + {"Ubuntu:", false}, + {"Ubuntu:LTS", false}, + {"Ubuntu:Pro", false}, + {"Ubuntu:Pro:LTS", false}, + {"Ubuntu::22.04", false}, + {"Ubuntu:22.04:", false}, + {"Ubuntu: 22.04", false}, + {"Ubuntu: ", false}, + {"Debian:12", false}, + } + + for _, tc := range tests { + if got := IsValidUbuntuReleaseEcosystem(tc.input); got != tc.want { + t.Errorf("IsValidUbuntuReleaseEcosystem(%q) = %v, want %v", tc.input, got, tc.want) + } + } +} From 280206a37b8374acba27413c00c0182538270c2f Mon Sep 17 00:00:00 2001 From: Rex P Date: Wed, 7 Oct 2026 11:48:54 +1100 Subject: [PATCH 3/4] feat(bindings/go): add ExperimentalQueryUbuntuPackageMapping client method --- bindings/go/osvdev/osvdev.go | 65 ++++++++++++++ bindings/go/osvdev/osvdev_test.go | 143 ++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) diff --git a/bindings/go/osvdev/osvdev.go b/bindings/go/osvdev/osvdev.go index e67434b4076..9eca9dbbcf0 100644 --- a/bindings/go/osvdev/osvdev.go +++ b/bindings/go/osvdev/osvdev.go @@ -27,9 +27,15 @@ const ( // DetermineVersionEndpoint is the URL for posting determineversion queries to OSV. DetermineVersionEndpoint = "/v1experimental/determineversion" + // UbuntuBinaryToSourceEndpoint is the URL for posting Ubuntu binary-to-source package mapping queries to OSV. + UbuntuBinaryToSourceEndpoint = "/v1experimental/ubuntu/binary-to-source" + // MaxQueriesPerQueryBatchRequest is a limit set in osv.dev's API, so is not configurable MaxQueriesPerQueryBatchRequest = 1000 + // MaxPackagesPerUbuntuMappingRequest is a limit set in osv.dev's API, so is not configurable + MaxPackagesPerUbuntuMappingRequest = 1000 + DefaultBaseURL = "https://api.osv.dev" ) @@ -183,6 +189,65 @@ func (c *OSVClient) ExperimentalDetermineVersion(ctx context.Context, query *api return &result, nil } +// ExperimentalQueryUbuntuPackageMapping is an interface to the experimental +// /v1experimental/ubuntu/binary-to-source endpoint: +// https://google.github.io/osv.dev/post-v1-ubuntu-binary-to-source/ +// +// If more than MaxPackagesPerUbuntuMappingRequest binary names are provided, +// requests are automatically chunked and executed concurrently. +func (c *OSVClient) ExperimentalQueryUbuntuPackageMapping( + ctx context.Context, + params *api.UbuntuPackageMappingParameters, +) (*api.UbuntuPackageMappingResponse, error) { + if len(params.GetBinaryNames()) <= MaxPackagesPerUbuntuMappingRequest { + var result api.UbuntuPackageMappingResponse + if err := c.makeRequest(ctx, UbuntuBinaryToSourceEndpoint, params, &result); err != nil { + return nil, err + } + + return &result, nil + } + + nameChunks := chunkBy(params.GetBinaryNames(), MaxPackagesPerUbuntuMappingRequest) + batchedResults := make([][]*api.SourcePackages, len(nameChunks)) + + g, errGrpCtx := errgroup.WithContext(ctx) + g.SetLimit(c.Config.MaxConcurrentBatchRequests) + for batchIndex, chunk := range nameChunks { + g.Go(func() error { + if errGrpCtx.Err() != nil { + return errGrpCtx.Err() + } + + var chunkResp api.UbuntuPackageMappingResponse + err := c.makeRequest(errGrpCtx, UbuntuBinaryToSourceEndpoint, &api.UbuntuPackageMappingParameters{ + Ecosystem: params.GetEcosystem(), + BinaryNames: chunk, + }, &chunkResp) + if err != nil { + return err + } + + batchedResults[batchIndex] = chunkResp.GetResults() + + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + + totalResp := &api.UbuntuPackageMappingResponse{ + Results: make([]*api.SourcePackages, 0, len(params.GetBinaryNames())), + } + for _, batch := range batchedResults { + totalResp.Results = append(totalResp.Results, batch...) + } + + return totalResp, nil +} + func (c *OSVClient) makeRequest(ctx context.Context, endpoint string, reqBody, resBody proto.Message) error { requestBytes, err := protojson.Marshal(reqBody) if err != nil { diff --git a/bindings/go/osvdev/osvdev_test.go b/bindings/go/osvdev/osvdev_test.go index e74b8977dde..c0df83db193 100644 --- a/bindings/go/osvdev/osvdev_test.go +++ b/bindings/go/osvdev/osvdev_test.go @@ -2,6 +2,10 @@ package osvdev_test import ( "context" + "fmt" + "io" + "net/http" + "net/http/httptest" "testing" "time" @@ -9,6 +13,8 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "github.com/ossf/osv-schema/bindings/go/osvconstants" "github.com/ossf/osv-schema/bindings/go/osvschema" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/testing/protocmp" "osv.dev/bindings/go/api" "osv.dev/bindings/go/internal/testhelper" "osv.dev/bindings/go/osvdev" @@ -445,3 +451,140 @@ func TestOSVClient_ExperimentalDetermineVersion(t *testing.T) { }) } } + +func TestOSVClient_ExperimentalQueryUbuntuPackageMapping(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != osvdev.UbuntuBinaryToSourceEndpoint { + http.Error(w, "unexpected path", http.StatusNotFound) + return + } + if r.Method != http.MethodPost { + http.Error(w, "unexpected method", http.StatusMethodNotAllowed) + return + } + + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var params api.UbuntuPackageMappingParameters + if err := protojson.Unmarshal(body, ¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if params.GetEcosystem() == "" { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":3,"message":"ecosystem is required"}`)) + + return + } + if len(params.GetBinaryNames()) > osvdev.MaxPackagesPerUbuntuMappingRequest { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"code":3,"message":"too many binary_names"}`)) + + return + } + + resp := &api.UbuntuPackageMappingResponse{ + Results: make([]*api.SourcePackages, len(params.GetBinaryNames())), + } + for i, bin := range params.GetBinaryNames() { + if bin == "unknown" { + resp.Results[i] = &api.SourcePackages{} + } else { + resp.Results[i] = &api.SourcePackages{ + SourceNames: []string{"src-" + bin}, + } + } + } + + out, err := protojson.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(out) + })) + t.Cleanup(server.Close) + + t.Run("single batch request", func(t *testing.T) { + t.Parallel() + + c := osvdev.DefaultClient() + c.BaseHostURL = server.URL + + got, err := c.ExperimentalQueryUbuntuPackageMapping(context.Background(), &api.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:24.04:LTS", + BinaryNames: []string{"libcurl4", "unknown"}, + }) + if err != nil { + t.Fatalf("ExperimentalQueryUbuntuPackageMapping failed: %v", err) + } + + want := &api.UbuntuPackageMappingResponse{ + Results: []*api.SourcePackages{ + {SourceNames: []string{"src-libcurl4"}}, + {}, + }, + } + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("ExperimentalQueryUbuntuPackageMapping mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("automatic chunking over 1000 packages", func(t *testing.T) { + t.Parallel() + + c := osvdev.DefaultClient() + c.BaseHostURL = server.URL + + const total = 1250 + binaryNames := make([]string, total) + wantResults := make([]*api.SourcePackages, total) + for i := range total { + name := fmt.Sprintf("pkg-%d", i) + binaryNames[i] = name + wantResults[i] = &api.SourcePackages{ + SourceNames: []string{"src-" + name}, + } + } + + got, err := c.ExperimentalQueryUbuntuPackageMapping(context.Background(), &api.UbuntuPackageMappingParameters{ + Ecosystem: "Ubuntu:22.04", + BinaryNames: binaryNames, + }) + if err != nil { + t.Fatalf("ExperimentalQueryUbuntuPackageMapping chunked failed: %v", err) + } + + want := &api.UbuntuPackageMappingResponse{ + Results: wantResults, + } + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("ExperimentalQueryUbuntuPackageMapping chunked mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("invalid request error propagation", func(t *testing.T) { + t.Parallel() + + c := osvdev.DefaultClient() + c.BaseHostURL = server.URL + + _, err := c.ExperimentalQueryUbuntuPackageMapping(context.Background(), &api.UbuntuPackageMappingParameters{ + BinaryNames: []string{"libcurl4"}, + }) + wantErr := testhelper.ErrContainsStr{ + Str: `client error: status="400 Bad Request" body={"code":3,"message":"ecosystem is required"}`, + } + if diff := cmp.Diff(wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want +got):\n%s", diff) + } + }) +} From 4180048f5f183a7e640d714286b6a98b845f53a8 Mon Sep 17 00:00:00 2001 From: Rex P Date: Wed, 7 Oct 2026 13:13:11 +1100 Subject: [PATCH 4/4] fix(bindings/go): guard nil params and zero MaxConcurrentBatchRequests --- bindings/go/osvdev/osvdev.go | 8 +++++++- bindings/go/osvdev/osvdev_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/bindings/go/osvdev/osvdev.go b/bindings/go/osvdev/osvdev.go index 9eca9dbbcf0..d516e5d8df5 100644 --- a/bindings/go/osvdev/osvdev.go +++ b/bindings/go/osvdev/osvdev.go @@ -199,6 +199,10 @@ func (c *OSVClient) ExperimentalQueryUbuntuPackageMapping( ctx context.Context, params *api.UbuntuPackageMappingParameters, ) (*api.UbuntuPackageMappingResponse, error) { + if params == nil { + return nil, errors.New("params cannot be nil") + } + if len(params.GetBinaryNames()) <= MaxPackagesPerUbuntuMappingRequest { var result api.UbuntuPackageMappingResponse if err := c.makeRequest(ctx, UbuntuBinaryToSourceEndpoint, params, &result); err != nil { @@ -212,7 +216,9 @@ func (c *OSVClient) ExperimentalQueryUbuntuPackageMapping( batchedResults := make([][]*api.SourcePackages, len(nameChunks)) g, errGrpCtx := errgroup.WithContext(ctx) - g.SetLimit(c.Config.MaxConcurrentBatchRequests) + if c.Config.MaxConcurrentBatchRequests > 0 { + g.SetLimit(c.Config.MaxConcurrentBatchRequests) + } for batchIndex, chunk := range nameChunks { g.Go(func() error { if errGrpCtx.Err() != nil { diff --git a/bindings/go/osvdev/osvdev_test.go b/bindings/go/osvdev/osvdev_test.go index c0df83db193..f7b700ddc7a 100644 --- a/bindings/go/osvdev/osvdev_test.go +++ b/bindings/go/osvdev/osvdev_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "sync/atomic" "testing" "time" @@ -455,6 +456,8 @@ func TestOSVClient_ExperimentalDetermineVersion(t *testing.T) { func TestOSVClient_ExperimentalQueryUbuntuPackageMapping(t *testing.T) { t.Parallel() + var chunkedRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != osvdev.UbuntuBinaryToSourceEndpoint { http.Error(w, "unexpected path", http.StatusNotFound) @@ -489,6 +492,9 @@ func TestOSVClient_ExperimentalQueryUbuntuPackageMapping(t *testing.T) { return } + if params.GetEcosystem() == "Ubuntu:22.04" { + chunkedRequests.Add(1) + } resp := &api.UbuntuPackageMappingResponse{ Results: make([]*api.SourcePackages, len(params.GetBinaryNames())), @@ -562,6 +568,9 @@ func TestOSVClient_ExperimentalQueryUbuntuPackageMapping(t *testing.T) { if err != nil { t.Fatalf("ExperimentalQueryUbuntuPackageMapping chunked failed: %v", err) } + if gotReqs := chunkedRequests.Load(); gotReqs != 2 { + t.Errorf("expected 2 chunked requests for %d packages, got %d", total, gotReqs) + } want := &api.UbuntuPackageMappingResponse{ Results: wantResults, @@ -571,6 +580,21 @@ func TestOSVClient_ExperimentalQueryUbuntuPackageMapping(t *testing.T) { } }) + t.Run("nil params error", func(t *testing.T) { + t.Parallel() + + c := osvdev.DefaultClient() + c.BaseHostURL = server.URL + + _, err := c.ExperimentalQueryUbuntuPackageMapping(context.Background(), nil) + wantErr := testhelper.ErrContainsStr{ + Str: "params cannot be nil", + } + if diff := cmp.Diff(wantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want +got):\n%s", diff) + } + }) + t.Run("invalid request error propagation", func(t *testing.T) { t.Parallel()