SPM-2105: store in DB if system was uploaded with cache refreshed

base/models/models.go

@@ -68,6 +68,7 @@ type SystemPlatform struct {
 	BaselineUpToDate      *bool  `gorm:"column:baseline_uptodate"`
 	YumUpdates            []byte `gorm:"column:yum_updates"`
 	SatelliteManaged      bool   `gorm:"column:satellite_managed"`
+	BuiltPkgcache         bool   `gorm:"column:built_pkgcache"`
 }
 
 func (SystemPlatform) TableName() string {

base/vmaas/vmaas.go

@@ -46,6 +46,7 @@ type UpdatesV3Response struct {
 	Releasever     *string                                 `json:"releasever,omitempty"`
 	Basearch       *string                                 `json:"basearch,omitempty"`
 	LastChange     *string                                 `json:"last_change,omitempty"`
+	BuildPkgcache  *bool                                   `json:"build_pkgcache,omitempty"`
 }
 
 // GetUpdateList returns the UpdateList field value if set, zero value otherwise.
@@ -57,6 +58,14 @@ func (o *UpdatesV3Response) GetUpdateList() map[string]UpdatesV3ResponseUpdateLi
 	return *o.UpdateList
 }
 
+// GetBuildPkgcache returns the boolean value for the `build_pkgcache` field of yum_updates
+func (o *UpdatesV3Response) GetBuildPkgcache() bool {
+	if o == nil || o.BuildPkgcache == nil {
+		return false
+	}
+	return *o.BuildPkgcache
+}
+
 type UpdatesV3ResponseUpdateList struct {
 	AvailableUpdates *[]UpdatesV3ResponseAvailableUpdates `json:"available_updates,omitempty"`
 }

database_admin/migrations/114_built_pkgcache.down.sql

@@ -0,0 +1 @@
+ALTER TABLE system_platform DROP COLUMN built_pkgcache;

database_admin/migrations/114_built_pkgcache.up.sql

@@ -0,0 +1 @@
+ALTER TABLE system_platform ADD COLUMN built_pkgcache BOOLEAN NOT NULL DEFAULT FALSE;

database_admin/schema/create_schema.sql

@@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS schema_migrations
 
 
 INSERT INTO schema_migrations
-VALUES (113, false);
+VALUES (114, false);
 
 -- ---------------------------------------------------------------------------
 -- Functions
@@ -710,6 +710,7 @@ CREATE TABLE IF NOT EXISTS system_platform
     applicable_advisory_bug_count_cache  INT                      NOT NULL DEFAULT 0,
     applicable_advisory_sec_count_cache  INT                      NOT NULL DEFAULT 0,
     satellite_managed                    BOOLEAN                  NOT NULL DEFAULT FALSE,
+    built_pkgcache                       BOOLEAN                  NOT NULL DEFAULT FALSE,
     PRIMARY KEY (rh_account_id, id),
     UNIQUE (rh_account_id, inventory_id),
     CONSTRAINT reporter_id FOREIGN KEY (reporter_id) REFERENCES reporter (id),

docs/v3/openapi.json

@@ -7196,6 +7196,9 @@
                     "basearch": {
                         "type": "string"
                     },
+                    "build_pkgcache": {
+                        "type": "boolean"
+                    },
                     "last_change": {
                         "type": "string"
                     },

listener/common_test.go

@@ -131,15 +131,16 @@ func assertSystemReposInDB(t *testing.T, systemID int64, repos []string) {
 	assert.Equal(t, c, int64(len(repos)))
 }
 
-func assertYumUpdatesInDB(t *testing.T, inventoryID string, yumUpdates []byte) {
+func assertYumUpdatesInDB(t *testing.T, inventoryID string, yumUpdates *YumUpdates) {
 	var system models.SystemPlatform
 	assert.NoError(t, database.Db.Where("inventory_id = ?::uuid", inventoryID).Find(&system).Error)
 	assert.Equal(t, system.InventoryID, inventoryID)
 	var systemYumUpdatesParsed vmaas.UpdatesV3Response
 	var yumUpdatesParsed vmaas.UpdatesV3Response
 	err := json.Unmarshal(system.YumUpdates, &systemYumUpdatesParsed)
 	assert.Nil(t, err)
-	err = json.Unmarshal(yumUpdates, &yumUpdatesParsed)
+	err = json.Unmarshal(yumUpdates.RawParsed, &yumUpdatesParsed)
 	assert.Nil(t, err)
 	assert.Equal(t, systemYumUpdatesParsed, yumUpdatesParsed)
+	assert.Equal(t, yumUpdates.BuiltPkgcache, system.BuiltPkgcache)
 }

listener/upload.go

@@ -77,6 +77,27 @@ type HostCustomMetadata struct {
 	YumUpdatesS3URL *string         `json:"yum_updates_s3url,omitempty"`
 }
 
+type YumUpdates struct {
+	RawParsed     json.RawMessage
+	BuiltPkgcache bool
+}
+
+// GetRawParsed returns prepared parsed raw yumupdates
+func (y *YumUpdates) GetRawParsed() json.RawMessage {
+	if y == nil {
+		return json.RawMessage{}
+	}
+	return y.RawParsed
+}
+
+// GetBuiltPkgcache returns boolean for build_pkgcache from yum_updates
+func (y *YumUpdates) GetBuiltPkgcache() bool {
+	if y == nil {
+		return false
+	}
+	return y.BuiltPkgcache
+}
+
 //nolint:funlen
 func HandleUpload(event HostEvent) error {
 	tStart := time.Now()
@@ -126,7 +147,7 @@ func HandleUpload(event HostEvent) error {
 		// don't fail, use vmaas evaluation
 		utils.LogError("err", err, "Could not get yum updates")
 	}
-	utils.LogTrace("inventoryID", event.Host.ID, "yum_updates", string(yumUpdates))
+	utils.LogTrace("inventoryID", event.Host.ID, "yum_updates", string(yumUpdates.GetRawParsed()))
 
 	if len(event.Host.SystemProfile.GetInstalledPackages()) == 0 && yumUpdates == nil {
 		utils.LogWarn("inventoryID", event.Host.ID, WarnSkippingNoPackages)
@@ -256,7 +277,7 @@ func updateReporterCounter(reporter string) {
 // nolint: funlen
 // Stores or updates base system profile, returing internal system id
 func updateSystemPlatform(tx *gorm.DB, inventoryID string, accountID int, host *Host,
-	yumUpdates []byte, updatesReq *vmaas.UpdatesV3Request) (*models.SystemPlatform, error) {
+	yumUpdates *YumUpdates, updatesReq *vmaas.UpdatesV3Request) (*models.SystemPlatform, error) {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, messagePartDuration.WithLabelValues("update-system-platform"))
 	updatesReqJSON, err := json.Marshal(updatesReq)
@@ -266,7 +287,7 @@ func updateSystemPlatform(tx *gorm.DB, inventoryID string, accountID int, host *
 
 	hash := sha256.Sum256(updatesReqJSON)
 	jsonChecksum := hex.EncodeToString(hash[:])
-	hash = sha256.Sum256(yumUpdates)
+	hash = sha256.Sum256(yumUpdates.GetRawParsed())
 	yumChecksum := hex.EncodeToString(hash[:])
 
 	var colsToUpdate = []string{
@@ -277,6 +298,7 @@ func updateSystemPlatform(tx *gorm.DB, inventoryID string, accountID int, host *
 		"stale_warning_timestamp",
 		"culled_timestamp",
 		"satellite_managed",
+		"built_pkgcache",
 	}
 
 	now := time.Now()
@@ -299,8 +321,9 @@ func updateSystemPlatform(tx *gorm.DB, inventoryID string, accountID int, host *
 		CulledTimestamp:       host.CulledTimestamp.Time(),
 		Stale:                 staleWarning != nil && staleWarning.Before(time.Now()),
 		ReporterID:            getReporterID(host.Reporter),
-		YumUpdates:            yumUpdates,
+		YumUpdates:            yumUpdates.GetRawParsed(),
 		SatelliteManaged:      host.SystemProfile.SatelliteManaged,
+		BuiltPkgcache:         yumUpdates.GetBuiltPkgcache(),
 	}
 
 	var oldChecksums map[string]string
@@ -535,7 +558,7 @@ func processModules(systemProfile *inventory.SystemProfile) *[]vmaas.UpdatesV3Re
 }
 
 // We have received new upload, update stored host data, and re-evaluate the host against VMaaS
-func processUpload(host *Host, yumUpdates []byte) (*models.SystemPlatform, error) {
+func processUpload(host *Host, yumUpdates *YumUpdates) (*models.SystemPlatform, error) {
 	tStart := time.Now()
 	defer utils.ObserveSecondsSince(tStart, messagePartDuration.WithLabelValues("upload-processing"))
 	// Ensure we have account stored
@@ -589,8 +612,9 @@ func processUpload(host *Host, yumUpdates []byte) (*models.SystemPlatform, error
 	return sys, nil
 }
 
-func getYumUpdates(event HostEvent, client *api.Client) ([]byte, error) {
+func getYumUpdates(event HostEvent, client *api.Client) (*YumUpdates, error) {
 	var parsed vmaas.UpdatesV3Response
+	res := &YumUpdates{}
 	yumUpdates := event.PlatformMetadata.CustomMetadata.YumUpdates
 	yumUpdatesURL := event.PlatformMetadata.CustomMetadata.YumUpdatesS3URL
 
@@ -615,7 +639,9 @@ func getYumUpdates(event HostEvent, client *api.Client) ([]byte, error) {
 	updatesMap := parsed.GetUpdateList()
 	if len(updatesMap) == 0 {
 		// system does not have any yum updates
-		return yumUpdates, nil
+		res.RawParsed = yumUpdates
+		res.BuiltPkgcache = parsed.GetBuildPkgcache()
+		return res, nil
 	}
 	// we need to get all packages to show up-to-date packages
 	installedPkgs := event.Host.SystemProfile.GetInstalledPackages()
@@ -633,7 +659,10 @@ func getYumUpdates(event HostEvent, client *api.Client) ([]byte, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to marshall yum updates")
 	}
-	return yumUpdates, nil
+	res.RawParsed = yumUpdates
+	res.BuiltPkgcache = parsed.GetBuildPkgcache()
+
+	return res, nil
 }
 
 func (host *Host) GetOrgID() string {

listener/upload_test.go

@@ -296,7 +296,7 @@ func TestUpdateSystemPlatformYumUpdates(t *testing.T) {
 	assertYumUpdatesInDB(t, id, yumUpdates)
 
 	// check that yumUpdates has been updated
-	yumUpdates = []byte("{}")
+	yumUpdates.RawParsed = []byte("{}")
 	_, err = updateSystemPlatform(database.Db, id, accountID1, createTestInvHost(t), yumUpdates, &req)
 	assert.Nil(t, err)
 	assertYumUpdatesInDB(t, id, yumUpdates)

platform/platform.go

@@ -142,6 +142,7 @@ var yumUpdates = `{
 			]
 		}
 	},
+	"build_pkgcache": false,
 	"metadata_time": "2022-05-30T14:00:25Z"
 }`