Skip to content

Commit 8c123f6

Browse files
wagoodmanwagoodman
committed
moderate review comments
Signed-off-by: Alex Goodman <[email protected]>
1 parent 0f7cc1e commit 8c123f6

File tree

7 files changed

+110
-100
lines changed

7 files changed

+110
-100
lines changed

‎DEVELOPING.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ well as acceptance testing. You will require the following:
77

88
- Python 3.8+ installed on your system. Consider using [pyenv](https://github.com/pyenv/pyenv) if you do not have a
99
preference for managing python interpreter installations.
10-
10+
- `zstd` binary utility if you are packaging v6+ DB schemas
11+
- _(optional)_ `xz` binary utility if you have specifically overridden the package command options
1112

1213
- [Poetry](https://python-poetry.org/) installed for dependency and virtualenv management for python dependencies, to install:
1314

‎README.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ curl -sSfL https://raw.githubusercontent.com/anchore/grype-db/main/install.sh |
2121
curl -sSfL https://raw.githubusercontent.com/anchore/grype-db/main/install.sh | sh -s -- -b <DESTINATION_DIR> <RELEASE_VERSION>
2222
```
2323

24+
> [!IMPORTANT]
25+
> You will require the `zstd` utility installed on your system to support the `package` command.
2426
2527
## Usage
2628

@@ -39,6 +41,7 @@ grype-db pull [-g] [-p PROVIDER ...]
3941
grype-db build [-g] [--dir=DIR] [--schema=SCHEMA] [--skip-validation] [-p PROVIDER ...]
4042

4143
# Package the already built DB file into an archive ready for upload and serving
44+
# note: you will require the zstd utility to be installed on your system
4245
grype-db package [--dir=DIR] [--publish-base-url=URL]
4346
```
4447

@@ -54,7 +57,7 @@ is created that is used in packaging and curation of the database file by this a
5457
and a `provider-metadata.json` file is created that includes the last successful run date for each provider.
5558
Use `-g` to generate the list of providers to pull based on the output of "vunnel list".
5659

57-
The `package` command archives the `vulnerability.db`, `metadata.json` and `provider-metadata.json` files into a `tar.gz` file. Additionally, a `listing.json`
60+
The `package` command archives the `vulnerability.db` file into a `tar.zstd` file. Additionally, a `latest.json`
5861
is generated to aid in serving one or more database archives for downstream consumption, where the consuming application should
5962
use the listing file to discover available archives available for download. The base URL used to create the download URL for each
6063
database archive is controlled by the `package.base-url` configuration option.

‎cmd/grype-db/cli/commands/build.go

+1-30
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"errors"
55
"fmt"
66
"os"
7-
"time"
87

98
"github.com/scylladb/go-set/strset"
109
"github.com/spf13/cobra"
@@ -108,7 +107,7 @@ func runBuild(cfg buildConfig) error {
108107
return fmt.Errorf("unable to get provider states: %w", err)
109108
}
110109

111-
earliest, err := earliestTimestamp(states)
110+
earliest, err := provider.States(states).EarliestTimestamp()
112111
if err != nil {
113112
return fmt.Errorf("unable to get earliest timestamp: %w", err)
114113
}
@@ -152,31 +151,3 @@ func providerStates(skipValidation bool, providers []provider.Provider) ([]provi
152151
}
153152
return states, nil
154153
}
155-
156-
func earliestTimestamp(states []provider.State) (time.Time, error) {
157-
if len(states) == 0 {
158-
return time.Time{}, fmt.Errorf("cannot find earliest timestamp: no states provided")
159-
}
160-
var earliest time.Time
161-
for _, s := range states {
162-
// the NVD api is constantly down, so we don't want to consider it for the earliest timestamp
163-
if s.Provider == "nvd" {
164-
log.WithFields("provider", s.Provider).Debug("not considering data age for provider")
165-
continue
166-
}
167-
if earliest.IsZero() {
168-
earliest = s.Timestamp
169-
continue
170-
}
171-
if s.Timestamp.Before(earliest) {
172-
earliest = s.Timestamp
173-
}
174-
}
175-
176-
if earliest.IsZero() {
177-
return time.Time{}, fmt.Errorf("unable to determine earliest timestamp")
178-
}
179-
180-
log.WithFields("timestamp", earliest).Debug("earliest data timestamp")
181-
return earliest, nil
182-
}

‎internal/tarutil/writer.go

+9
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,15 @@ func newShellCompressor(c string, archive io.Writer) (*shellCompressor, error) {
7373
return nil, fmt.Errorf("unable to parse command: %w", err)
7474
}
7575
binary := args[0]
76+
77+
binPath, err := exec.LookPath(binary)
78+
if err != nil {
79+
return nil, fmt.Errorf("unable to find binary %q: %w", binary, err)
80+
}
81+
if binPath == "" {
82+
return nil, fmt.Errorf("unable to find binary %q in PATH", binary)
83+
}
84+
7685
args = args[1:]
7786
cmd := exec.Command(binary, args...)
7887
log.Debug(strings.Join(cmd.Args, " "))

‎pkg/process/package.go

+57-57
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,11 @@ import (
99
"strings"
1010
"time"
1111

12+
"github.com/scylladb/go-set/strset"
13+
1214
"github.com/anchore/grype-db/internal/log"
1315
"github.com/anchore/grype-db/internal/tarutil"
16+
"github.com/anchore/grype-db/pkg/provider"
1417
grypeDBLegacyDistribution "github.com/anchore/grype/grype/db/legacy/distribution"
1518
v6 "github.com/anchore/grype/grype/db/v6"
1619
v6Distribution "github.com/anchore/grype/grype/db/v6/distribution"
@@ -31,69 +34,34 @@ func packageDB(dbDir, overrideArchiveExtension string) error {
3134
}
3235
log.WithFields("from", dbDir, "extension", extension).Info("packaging database")
3336

34-
tarPath, err := calculateTarPath(dbDir, extension)
35-
if err != nil {
36-
return err
37-
}
38-
39-
if err := populateTar(tarPath); err != nil {
40-
return err
41-
}
42-
43-
log.WithFields("path", tarPath).Info("created database archive")
44-
45-
return writeLatestDocument(tarPath)
46-
}
47-
48-
func resolveExtension(overrideArchiveExtension string) (string, error) {
49-
var extension = "tar.zst"
50-
51-
if overrideArchiveExtension != "" {
52-
extension = strings.TrimLeft(overrideArchiveExtension, ".")
53-
}
54-
55-
var found bool
56-
for _, valid := range []string{"tar.zst", "tar.xz", "tar.gz"} {
57-
if valid == extension {
58-
found = true
59-
break
60-
}
61-
}
62-
63-
if !found {
64-
return "", fmt.Errorf("unsupported archive extension %q", extension)
65-
}
66-
return extension, nil
67-
}
68-
69-
func calculateTarPath(dbDir string, extension string) (string, error) {
7037
s, err := v6.NewReader(v6.Config{DBDirPath: dbDir})
7138
if err != nil {
72-
return "", fmt.Errorf("unable to open vulnerability store: %w", err)
39+
return fmt.Errorf("unable to open vulnerability store: %w", err)
7340
}
7441

7542
metadata, err := s.GetDBMetadata()
76-
if err != nil {
77-
return "", fmt.Errorf("unable to get vulnerability store metadata: %w", err)
43+
if err != nil || metadata == nil {
44+
return fmt.Errorf("unable to get vulnerability store metadata: %w", err)
7845
}
7946

8047
if metadata.Model != v6.ModelVersion {
81-
return "", fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model)
48+
return fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model)
8249
}
8350

84-
providers, err := s.AllProviders()
51+
providerModels, err := s.AllProviders()
8552
if err != nil {
86-
return "", fmt.Errorf("unable to get all providers: %w", err)
53+
return fmt.Errorf("unable to get all providers: %w", err)
8754
}
8855

89-
if len(providers) == 0 {
90-
return "", fmt.Errorf("no providers found in the vulnerability store")
56+
if len(providerModels) == 0 {
57+
return fmt.Errorf("no providers found in the vulnerability store")
9158
}
9259

93-
eldest := eldestProviderTimestamp(providers)
94-
if eldest == nil {
95-
return "", errors.New("could not resolve eldest provider timestamp")
60+
eldest, err := toProviders(providerModels).EarliestTimestamp()
61+
if err != nil {
62+
return err
9663
}
64+
9765
// output archive vulnerability-db_VERSION_OLDESTDATADATE_BUILTEPOCH.tar.gz, where:
9866
// - VERSION: schema version in the form of v#.#.#
9967
// - OLDESTDATADATE: RFC3338 formatted value of the oldest date capture date found for all contained providers
@@ -106,19 +74,51 @@ func calculateTarPath(dbDir string, extension string) (string, error) {
10674
extension,
10775
)
10876

109-
return filepath.Join(dbDir, tarName), err
77+
tarPath := filepath.Join(dbDir, tarName)
78+
79+
if err := populateTar(tarPath); err != nil {
80+
return err
81+
}
82+
83+
log.WithFields("path", tarPath).Info("created database archive")
84+
85+
return writeLatestDocument(tarPath, *metadata)
86+
}
87+
88+
func toProviders(states []v6.Provider) provider.States {
89+
var result provider.States
90+
for _, state := range states {
91+
result = append(result, provider.State{
92+
Provider: state.ID,
93+
Timestamp: *state.DateCaptured,
94+
})
95+
}
96+
return result
11097
}
11198

112-
func eldestProviderTimestamp(providers []v6.Provider) *time.Time {
113-
var eldest *time.Time
114-
for _, p := range providers {
115-
if eldest == nil || p.DateCaptured.Before(*eldest) {
116-
eldest = p.DateCaptured
99+
func resolveExtension(overrideArchiveExtension string) (string, error) {
100+
var extension = "tar.zst"
101+
102+
if overrideArchiveExtension != "" {
103+
extension = strings.TrimLeft(overrideArchiveExtension, ".")
104+
}
105+
106+
var found bool
107+
for _, valid := range []string{"tar.zst", "tar.xz", "tar.gz"} {
108+
if valid == extension {
109+
found = true
110+
break
117111
}
118112
}
119-
return eldest
113+
114+
if !found {
115+
return "", fmt.Errorf("unsupported archive extension %q", extension)
116+
}
117+
return extension, nil
120118
}
121119

120+
var listingFiles = strset.New("listing.json", "latest.json", "history.json")
121+
122122
func populateTar(tarPath string) error {
123123
originalDir, err := os.Getwd()
124124
if err != nil {
@@ -146,7 +146,7 @@ func populateTar(tarPath string) error {
146146

147147
var files []string
148148
for _, fi := range fileInfos {
149-
if fi.Name() != "listing.json" && !strings.Contains(fi.Name(), ".tar.") {
149+
if !listingFiles.Has(fi.Name()) && !strings.Contains(fi.Name(), ".tar.") {
150150
files = append(files, fi.Name())
151151
}
152152
}
@@ -158,8 +158,8 @@ func populateTar(tarPath string) error {
158158
return nil
159159
}
160160

161-
func writeLatestDocument(tarPath string) error {
162-
archive, err := v6Distribution.NewArchive(tarPath)
161+
func writeLatestDocument(tarPath string, metadata v6.DBMetadata) error {
162+
archive, err := v6Distribution.NewArchive(tarPath, *metadata.BuildTimestamp, metadata.Model, metadata.Revision, metadata.Addition)
163163
if err != nil || archive == nil {
164164
return fmt.Errorf("unable to create archive: %w", err)
165165
}

‎pkg/provider/state.go

+28
Original file line numberDiff line numberDiff line change
@@ -134,3 +134,31 @@ func (s States) Names() []string {
134134
}
135135
return names
136136
}
137+
138+
func (s States) EarliestTimestamp() (time.Time, error) {
139+
if len(s) == 0 {
140+
return time.Time{}, fmt.Errorf("cannot find earliest timestamp: no states provided")
141+
}
142+
var earliest time.Time
143+
for _, curState := range s {
144+
// the NVD api is constantly down, so we don't want to consider it for the earliest timestamp
145+
if curState.Provider == "nvd" {
146+
log.WithFields("provider", curState.Provider).Debug("not considering data age for provider")
147+
continue
148+
}
149+
if earliest.IsZero() {
150+
earliest = curState.Timestamp
151+
continue
152+
}
153+
if curState.Timestamp.Before(earliest) {
154+
earliest = curState.Timestamp
155+
}
156+
}
157+
158+
if earliest.IsZero() {
159+
return time.Time{}, fmt.Errorf("unable to determine earliest timestamp")
160+
}
161+
162+
log.WithFields("timestamp", earliest).Debug("earliest data timestamp")
163+
return earliest, nil
164+
}

‎cmd/grype-db/cli/commands/build_test.go renamed to ‎pkg/provider/state_test.go

+9-11
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,23 @@
1-
package commands
1+
package provider
22

33
import (
44
"reflect"
55
"testing"
66
"time"
77

88
"github.com/stretchr/testify/require"
9-
10-
"github.com/anchore/grype-db/pkg/provider"
119
)
1210

1311
func Test_earliestTimestamp(t *testing.T) {
1412
tests := []struct {
1513
name string
16-
states []provider.State
14+
states []State
1715
want time.Time
1816
wantErr require.ErrorAssertionFunc
1917
}{
2018
{
2119
name: "happy path",
22-
states: []provider.State{
20+
states: []State{
2321
{
2422
Timestamp: time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC),
2523
},
@@ -34,13 +32,13 @@ func Test_earliestTimestamp(t *testing.T) {
3432
},
3533
{
3634
name: "empty states",
37-
states: []provider.State{},
35+
states: []State{},
3836
want: time.Time{},
3937
wantErr: requireErrorContains("cannot find earliest timestamp: no states provided"),
4038
},
4139
{
4240
name: "single state",
43-
states: []provider.State{
41+
states: []State{
4442
{
4543
Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
4644
},
@@ -49,7 +47,7 @@ func Test_earliestTimestamp(t *testing.T) {
4947
},
5048
{
5149
name: "all states have provider nvd",
52-
states: []provider.State{
50+
states: []State{
5351
{
5452
Provider: "nvd",
5553
Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
@@ -64,7 +62,7 @@ func Test_earliestTimestamp(t *testing.T) {
6462
},
6563
{
6664
name: "mix of nvd and non-nvd providers",
67-
states: []provider.State{
65+
states: []State{
6866
{
6967
Provider: "nvd",
7068
Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
@@ -82,7 +80,7 @@ func Test_earliestTimestamp(t *testing.T) {
8280
},
8381
{
8482
name: "timestamps are the same",
85-
states: []provider.State{
83+
states: []State{
8684
{
8785
Timestamp: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
8886
},
@@ -102,7 +100,7 @@ func Test_earliestTimestamp(t *testing.T) {
102100
if tt.wantErr == nil {
103101
tt.wantErr = require.NoError
104102
}
105-
got, err := earliestTimestamp(tt.states)
103+
got, err := States(tt.states).EarliestTimestamp()
106104
tt.wantErr(t, err)
107105
if err != nil {
108106
return

0 commit comments

Comments
 (0)