Skip to content

Commit

Permalink
Introduce minimal source coordinates (anchore#623)
Browse files Browse the repository at this point in the history 
* split source.Location and create source.Coordinates for minimal path addressing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* move coordinates into separate file

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* Update syft/source/coordinates.go

Co-authored-by: Dan Luhring <luhring@users.noreply.github.com>
  • Loading branch information
@wagoodman @luhring
wagoodman and luhring committed Nov 18, 2021
1 parent 5f5ae1a commit 372ef61
Show file tree
Hide file tree
Showing 31 changed files with 262 additions and 249 deletions.
12 changes: 8 additions & 4 deletions internal/anchore/import_package_sbom_test.go
View file
Expand Up @@ -59,8 +59,10 @@ func TestPackageSbomToModel(t *testing.T) {
FoundBy: "foundBy",
Locations: []source.Location{
{
RealPath: "path",
FileSystemID: "layerID",
Coordinates: source.Coordinates{
RealPath: "path",
FileSystemID: "layerID",
},
},
},
Licenses: []string{"license"},
Expand Down Expand Up @@ -157,8 +159,10 @@ func TestPackageSbomImport(t *testing.T) {
FoundBy: "foundBy",
Locations: []source.Location{
{
RealPath: "path",
FileSystemID: "layerID",
Coordinates: source.Coordinates{
RealPath: "path",
FileSystemID: "layerID",
},
},
},
Licenses: []string{"license"},
Expand Down
10 changes: 2 additions & 8 deletions internal/formats/common/spdxhelpers/source_info_test.go
View file
Expand Up @@ -19,14 +19,8 @@ func Test_SourceInfo(t *testing.T) {
input: pkg.Package{
// note: no type given
Locations: []source.Location{
{
RealPath: "/a-place",
VirtualPath: "/b-place",
},
{
RealPath: "/c-place",
VirtualPath: "/d-place",
},
source.NewVirtualLocation("/a-place", "/b-place"),
source.NewVirtualLocation("/c-place", "/d-place"),
},
},
expected: []string{
Expand Down
4 changes: 2 additions & 2 deletions internal/formats/common/testutils/utils.go
View file
Expand Up @@ -200,7 +200,7 @@ func newDirectoryCatalog() *pkg.Catalog {
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: []source.Location{
{RealPath: "/some/path/pkg1"},
source.NewLocation("/some/path/pkg1"),
},
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Expand All @@ -225,7 +225,7 @@ func newDirectoryCatalog() *pkg.Catalog {
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: []source.Location{
{RealPath: "/some/path/pkg1"},
source.NewLocation("/some/path/pkg1"),
},
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Expand Down
20 changes: 10 additions & 10 deletions internal/formats/syftjson/model/package.go
View file
Expand Up @@ -17,16 +17,16 @@ type Package struct {

// PackageBasicData contains non-ambiguous values (type-wise) from pkg.Package.
type PackageBasicData struct {
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"`
Locations []source.Location `json:"locations"`
Licenses []string `json:"licenses"`
Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"`
PURL string `json:"purl"`
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Type pkg.Type `json:"type"`
FoundBy string `json:"foundBy"`
Locations []source.Coordinates `json:"locations"`
Licenses []string `json:"licenses"`
Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"`
PURL string `json:"purl"`
}

// PackageCustomData contains ambiguous values (type-wise) from pkg.Package.
Expand Down
4 changes: 2 additions & 2 deletions internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden
View file
@@ -1,7 +1,7 @@
{
"artifacts": [
{
"id": "cbf4f3077fc7deee",
"id": "2a115ac97d018a0e",
"name": "package-1",
"version": "1.0.1",
"type": "python",
Expand Down Expand Up @@ -36,7 +36,7 @@
}
},
{
"id": "1a39aadd9705c2b9",
"id": "5e920b2bece2c3ae",
"name": "package-2",
"version": "2.0.1",
"type": "deb",
Expand Down
4 changes: 2 additions & 2 deletions internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden
View file
@@ -1,7 +1,7 @@
{
"artifacts": [
{
"id": "d1d433485a31ed07",
"id": "888661d4f0362f02",
"name": "package-1",
"version": "1.0.1",
"type": "python",
Expand Down Expand Up @@ -32,7 +32,7 @@
}
},
{
"id": "2db629ca48fa6786",
"id": "4068ff5e8926b305",
"name": "package-2",
"version": "2.0.1",
"type": "deb",
Expand Down
13 changes: 6 additions & 7 deletions internal/formats/syftjson/to_format_model.go
View file
Expand Up @@ -58,25 +58,24 @@ func toPackageModel(p pkg.Package) model.Package {
cpes[i] = c.BindToFmtString()
}

// ensure collections are never nil for presentation reasons
var locations = make([]source.Location, 0)
if p.Locations != nil {
locations = p.Locations
}

var licenses = make([]string, 0)
if p.Licenses != nil {
licenses = p.Licenses
}

var coordinates = make([]source.Coordinates, len(p.Locations))
for i, l := range p.Locations {
coordinates[i] = l.Coordinates
}

return model.Package{
PackageBasicData: model.PackageBasicData{
ID: string(p.ID()),
Name: p.Name,
Version: p.Version,
Type: p.Type,
FoundBy: p.FoundBy,
Locations: locations,
Locations: coordinates,
Licenses: licenses,
Language: p.Language,
CPEs: cpes,
Expand Down
7 changes: 6 additions & 1 deletion internal/formats/syftjson/to_syft_model.go
View file
Expand Up @@ -60,11 +60,16 @@ func toSyftPackage(p model.Package) pkg.Package {
cpes = append(cpes, value)
}

var locations = make([]source.Location, len(p.Locations))
for i, c := range p.Locations {
locations[i] = source.NewLocationFromCoordinates(c)
}

return pkg.Package{
Name: p.Name,
Version: p.Version,
FoundBy: p.FoundBy,
Locations: p.Locations,
Locations: locations,
Licenses: p.Licenses,
Language: p.Language,
Type: p.Type,
Expand Down
11 changes: 4 additions & 7 deletions internal/presenter/poweruser/json_file_classifications.go
View file
Expand Up @@ -8,26 +8,23 @@ import (
)

type JSONFileClassifications struct {
Location source.Location `json:"location"`
Location source.Coordinates `json:"location"`
Classification file.Classification `json:"classification"`
}

func NewJSONFileClassifications(data map[source.Location][]file.Classification) []JSONFileClassifications {
func NewJSONFileClassifications(data map[source.Coordinates][]file.Classification) []JSONFileClassifications {
results := make([]JSONFileClassifications, 0)
for location, classifications := range data {
for coordinates, classifications := range data {
for _, classification := range classifications {
results = append(results, JSONFileClassifications{
Location: location,
Location: coordinates,
Classification: classification,
})
}
}

// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath == results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results
Expand Down
13 changes: 5 additions & 8 deletions internal/presenter/poweruser/json_file_contents.go
View file
Expand Up @@ -7,24 +7,21 @@ import (
)

type JSONFileContents struct {
Location source.Location `json:"location"`
Contents string `json:"contents"`
Location source.Coordinates `json:"location"`
Contents string `json:"contents"`
}

func NewJSONFileContents(data map[source.Location]string) []JSONFileContents {
func NewJSONFileContents(data map[source.Coordinates]string) []JSONFileContents {
results := make([]JSONFileContents, 0)
for location, contents := range data {
for coordinates, contents := range data {
results = append(results, JSONFileContents{
Location: location,
Location: coordinates,
Contents: contents,
})
}

// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath == results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results
Expand Down
15 changes: 6 additions & 9 deletions internal/presenter/poweruser/json_file_metadata.go
View file
Expand Up @@ -11,7 +11,7 @@ import (
)

type JSONFileMetadata struct {
Location source.Location `json:"location"`
Location source.Coordinates `json:"location"`
Metadata JSONFileMetadataEntry `json:"metadata"`
}

Expand All @@ -25,21 +25,21 @@ type JSONFileMetadataEntry struct {
MIMEType string `json:"mimeType"`
}

func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests map[source.Location][]file.Digest) ([]JSONFileMetadata, error) {
func NewJSONFileMetadata(data map[source.Coordinates]source.FileMetadata, digests map[source.Coordinates][]file.Digest) ([]JSONFileMetadata, error) {
results := make([]JSONFileMetadata, 0)
for location, metadata := range data {
for coordinates, metadata := range data {
mode, err := strconv.Atoi(fmt.Sprintf("%o", metadata.Mode))
if err != nil {
return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", location, metadata.Mode, err)
return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", coordinates, metadata.Mode, err)
}

var digestResults []file.Digest
if digestsForLocation, exists := digests[location]; exists {
if digestsForLocation, exists := digests[coordinates]; exists {
digestResults = digestsForLocation
}

results = append(results, JSONFileMetadata{
Location: location,
Location: coordinates,
Metadata: JSONFileMetadataEntry{
Mode: mode,
Type: metadata.Type,
Expand All @@ -54,9 +54,6 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m

// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath == results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results, nil
Expand Down
28 changes: 16 additions & 12 deletions internal/presenter/poweruser/json_presenter_test.go
View file
Expand Up @@ -37,7 +37,9 @@ func TestJSONPresenter(t *testing.T) {
Version: "1.0.1",
Locations: []source.Location{
{
RealPath: "/a/place/a",
Coordinates: source.Coordinates{
RealPath: "/a/place/a",
},
},
},
Type: pkg.PythonPkg,
Expand All @@ -60,7 +62,9 @@ func TestJSONPresenter(t *testing.T) {
Version: "2.0.1",
Locations: []source.Location{
{
RealPath: "/b/place/b",
Coordinates: source.Coordinates{
RealPath: "/b/place/b",
},
},
},
Type: pkg.DebPkg,
Expand All @@ -86,49 +90,49 @@ func TestJSONPresenter(t *testing.T) {
cfg := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
FileMetadata: map[source.Location]source.FileMetadata{
source.NewLocation("/a/place"): {
FileMetadata: map[source.Coordinates]source.FileMetadata{
source.NewLocation("/a/place").Coordinates: {
Mode: 0775,
Type: "directory",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/a/place/a"): {
source.NewLocation("/a/place/a").Coordinates: {
Mode: 0775,
Type: "regularFile",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b"): {
source.NewLocation("/b").Coordinates: {
Mode: 0775,
Type: "symbolicLink",
LinkDestination: "/c",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b/place/b"): {
source.NewLocation("/b/place/b").Coordinates: {
Mode: 0644,
Type: "regularFile",
UserID: 1,
GroupID: 2,
},
},
FileDigests: map[source.Location][]file.Digest{
source.NewLocation("/a/place/a"): {
FileDigests: map[source.Coordinates][]file.Digest{
source.NewLocation("/a/place/a").Coordinates: {
{
Algorithm: "sha256",
Value: "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
},
},
source.NewLocation("/b/place/b"): {
source.NewLocation("/b/place/b").Coordinates: {
{
Algorithm: "sha256",
Value: "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c",
},
},
},
FileContents: map[source.Location]string{
source.NewLocation("/a/place/a"): "the-contents",
FileContents: map[source.Coordinates]string{
source.NewLocation("/a/place/a").Coordinates: "the-contents",
},
Distro: &distro.Distro{
Type: distro.RedHat,
Expand Down
13 changes: 5 additions & 8 deletions internal/presenter/poweruser/json_secrets.go
View file
Expand Up @@ -8,25 +8,22 @@ import (
)

type JSONSecrets struct {
Location source.Location `json:"location"`
Location source.Coordinates `json:"location"`
Secrets []file.SearchResult `json:"secrets"`
}

func NewJSONSecrets(data map[source.Location][]file.SearchResult) []JSONSecrets {
func NewJSONSecrets(data map[source.Coordinates][]file.SearchResult) []JSONSecrets {
results := make([]JSONSecrets, 0)
for location, secrets := range data {
for coordinates, secrets := range data {
results = append(results, JSONSecrets{
Location: location,
Location: coordinates,
Secrets: secrets,
})
}

// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath != results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return false
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results
}

0 comments on commit 372ef61

Please sign in to comment.