diff --git a/conf/local.env b/conf/local.env index bf34084df..6ac69592d 100644 --- a/conf/local.env +++ b/conf/local.env @@ -1,5 +1,6 @@ GIN_MODE=release LOG_LEVEL=DEBUG +DB_DEBUG=true USE_TESTING_DB=1 ACG_CONFIG=./conf/cdappconfig.json diff --git a/docs/v3/openapi.json b/docs/v3/openapi.json index 42e464286..0c1da90a1 100644 --- a/docs/v3/openapi.json +++ b/docs/v3/openapi.json @@ -141,11 +141,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -157,7 +162,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -434,11 +439,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -450,7 +460,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -1085,11 +1095,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -1101,7 +1116,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -1382,11 +1397,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -1398,7 +1418,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -1607,11 +1627,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -1623,7 +1648,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -1895,11 +1920,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -1911,7 +1941,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -2118,11 +2148,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -2134,7 +2169,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -2696,11 +2731,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -2712,7 +2752,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -3022,11 +3062,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -3038,7 +3083,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -3317,11 +3362,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -3333,7 +3383,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -3611,11 +3661,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -3627,7 +3682,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -3970,11 +4025,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -3986,7 +4046,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -4190,11 +4250,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -4206,7 +4271,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", @@ -4573,11 +4638,16 @@ } }, { - "name": "filter[group_name][in]", + "name": "filter[group_name]", "in": "query", "description": "Filter systems by inventory groups", + "style": "form", + "explode": true, "schema": { - "type": "string" + "type": "array", + "items": { + "type": "string" + } } }, { @@ -4589,7 +4659,7 @@ } }, { - "name": "filter[system_profile][sap_sids][in]", + "name": "filter[system_profile][sap_sids]", "in": "query", "description": "Filter systems by their SAP SIDs", "style": "form", diff --git a/manager/controllers/advisories.go b/manager/controllers/advisories.go index d9576a218..74979558b 100644 --- a/manager/controllers/advisories.go +++ b/manager/controllers/advisories.go @@ -122,7 +122,7 @@ func advisoriesCommon(c *gin.Context) (*gorm.DB, *ListMeta, []string, error) { account := c.GetInt(middlewares.KeyAccount) groups := c.GetStringMapString(middlewares.KeyInventoryGroups) var query *gorm.DB - filters, err := ParseInventoryFilters(c) + filters, inventoryFilters, err := ParseInventoryFilters(c, AdvisoriesOpts) if err != nil { return nil, nil, nil, err } @@ -137,9 +137,9 @@ func advisoriesCommon(c *gin.Context) (*gorm.DB, *ListMeta, []string, error) { validCache = false } } - if !validCache || HasInventoryFilter(c) || len(groups[rbac.KeyGrouped]) != 0 { + if !validCache || HasInventoryFilter(inventoryFilters) || len(groups[rbac.KeyGrouped]) != 0 { var err error - query = buildQueryAdvisoriesTagged(db, filters, account, groups) + query = buildQueryAdvisoriesTagged(db, inventoryFilters, account, groups) if err != nil { return nil, nil, nil, err } // Error handled in method itself @@ -147,7 +147,7 @@ func advisoriesCommon(c *gin.Context) (*gorm.DB, *ListMeta, []string, error) { query = buildQueryAdvisories(db, account) } - query, meta, params, err := ListCommon(query, c, filters, AdvisoriesOpts) + query, meta, params, err := ListCommon(query, c, filters, inventoryFilters, AdvisoriesOpts) // Error handling and setting of result code & content is done in ListCommon return query, meta, params, err } @@ -172,9 +172,9 @@ func advisoriesCommon(c *gin.Context) (*gorm.DB, *ListMeta, []string, error) { // @Param filter[installable_systems] query string false "Filter" // @Param filter[applicable_systems] query string false "Filter" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -240,9 +240,9 @@ func AdvisoriesListHandler(c *gin.Context) { // @Param filter[installable_systems] query string false "Filter" // @Param filter[applicable_systems] query string false "Filter" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" diff --git a/manager/controllers/advisories_export.go b/manager/controllers/advisories_export.go index 3518e5146..83b429dbd 100644 --- a/manager/controllers/advisories_export.go +++ b/manager/controllers/advisories_export.go @@ -30,7 +30,7 @@ import ( func AdvisoriesExportHandler(c *gin.Context) { account := c.GetInt(middlewares.KeyAccount) groups := c.GetStringMapString(middlewares.KeyInventoryGroups) - filters, err := ParseInventoryFilters(c) + _, inventoryFilters, err := ParseInventoryFilters(c, SystemOpts) if err != nil { return } @@ -47,9 +47,9 @@ func AdvisoriesExportHandler(c *gin.Context) { validCache = false } } - if !validCache || HasInventoryFilter(c) || len(groups[rbac.KeyGrouped]) != 0 { + if !validCache || HasInventoryFilter(inventoryFilters) || len(groups[rbac.KeyGrouped]) != 0 { var err error - query = buildQueryAdvisoriesTagged(db, filters, account, groups) + query = buildQueryAdvisoriesTagged(db, inventoryFilters, account, groups) if err != nil { return } // Error handled in method itself diff --git a/manager/controllers/advisories_test.go b/manager/controllers/advisories_test.go index dcbb7a04c..be4ae2d70 100644 --- a/manager/controllers/advisories_test.go +++ b/manager/controllers/advisories_test.go @@ -186,14 +186,6 @@ func TestAdvisoriesFilterTypeID4(t *testing.T) { } func TestAdvisoriesFilterTypeID5(t *testing.T) { - output := testAdvisories(t, "/?filter[advisory_type_name]!=other") - assert.Equal(t, 4, len(output.Data)) - for _, advisory := range output.Data { - assert.NotContains(t, "bugfix enhancement security", advisory.Attributes.AdvisoryTypeName) - } -} - -func TestAdvisoriesFilterTypeID6(t *testing.T) { output := testAdvisories(t, "/?filter[advisory_type_name]=in:other,bugfix") assert.Equal(t, 7, len(output.Data)) for _, advisory := range output.Data { diff --git a/manager/controllers/advisory_systems.go b/manager/controllers/advisory_systems.go index f377273fd..31c5f366c 100644 --- a/manager/controllers/advisory_systems.go +++ b/manager/controllers/advisory_systems.go @@ -58,16 +58,16 @@ func advisorySystemsCommon(c *gin.Context) (*gorm.DB, *ListMeta, []string, error } query := buildAdvisorySystemsQuery(db, account, groups, advisoryName, apiver) - filters, err := ParseInventoryFilters(c) - if err != nil { - return nil, nil, nil, err - } // Error handled in method itself - query, _ = ApplyInventoryFilter(filters, query, "sp.inventory_id") opts := AdvisorySystemOptsV3 if apiver < 3 { opts = AdvisorySystemOpts } - query, meta, params, err := ListCommon(query, c, filters, opts) + filters, inventoryFilters, err := ParseInventoryFilters(c, opts) + if err != nil { + return nil, nil, nil, err + } // Error handled in method itself + query, _ = ApplyInventoryFilter(inventoryFilters, query, "sp.inventory_id") + query, meta, params, err := ListCommon(query, c, filters, inventoryFilters, opts) // Error handled in method itself return query, meta, params, err } @@ -91,9 +91,9 @@ func advisorySystemsCommon(c *gin.Context) (*gorm.DB, *ListMeta, []string, error // @Param filter[template] query string false "Filter" // @Param filter[os] query string false "Filter OS version" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -171,9 +171,9 @@ func advisorySystemsListHandler(c *gin.Context) { // @Param filter[osmajor] query string false "Filter" // @Param filter[os] query string false "Filter OS version" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" diff --git a/manager/controllers/advisory_systems_export.go b/manager/controllers/advisory_systems_export.go index c94a5ee1a..a39b3ffad 100644 --- a/manager/controllers/advisory_systems_export.go +++ b/manager/controllers/advisory_systems_export.go @@ -22,9 +22,9 @@ import ( // @Param filter[id] query string false "Filter" // @Param filter[display_name] query string false "Filter" // @Param filter[stale] query string false "Filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -61,11 +61,11 @@ func AdvisorySystemsExportHandler(c *gin.Context) { } query := buildAdvisorySystemsQuery(db, account, groups, advisoryName, apiver) - filters, err := ParseInventoryFilters(c) + _, inventoryFilters, err := ParseInventoryFilters(c, AdvisorySystemOpts) if err != nil { return } // Error handled in method itself - query, _ = ApplyInventoryFilter(filters, query, "sp.inventory_id") + query, _ = ApplyInventoryFilter(inventoryFilters, query, "sp.inventory_id") query = query.Order("sp.id") query, err = ExportListCommon(query, c, AdvisorySystemOpts) diff --git a/manager/controllers/baseline_systems.go b/manager/controllers/baseline_systems.go index 22d58c3ee..21e7ac4a2 100644 --- a/manager/controllers/baseline_systems.go +++ b/manager/controllers/baseline_systems.go @@ -108,11 +108,11 @@ func queryBaselineSystems(c *gin.Context, account, apiver int, groups map[string } query := buildQueryBaselineSystems(db, account, groups, id, apiver) - filters, err := ParseInventoryFilters(c) + _, inventoryFilters, err := ParseInventoryFilters(c, BaselineSystemOpts) if err != nil { return nil, err } // Error handled in method itself - query, _ = ApplyInventoryFilter(filters, query, "sp.inventory_id") + query, _ = ApplyInventoryFilter(inventoryFilters, query, "sp.inventory_id") return query, nil } @@ -123,7 +123,12 @@ func baselineSystemsCommon(c *gin.Context, account, apiver int, groups map[strin return nil, nil, nil, err } // Error handled in method itself - query, meta, params, err := ListCommon(query, c, nil, BaselineSystemOpts) + filters, inventoryFilters, err := ParseInventoryFilters(c, PackageSystemsOpts) + if err != nil { + // Error handled in method itself + return nil, nil, nil, err + } + query, meta, params, err := ListCommon(query, c, filters, inventoryFilters, BaselineSystemOpts) if err != nil { // Error handling and setting of result code & content is done in ListCommon return nil, nil, nil, err @@ -147,9 +152,9 @@ func baselineSystemsCommon(c *gin.Context, account, apiver int, groups map[strin // @Param filter[display_name] query string false "Filter" // @Param filter[os] query string false "Filter" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" diff --git a/manager/controllers/baseline_systems_export.go b/manager/controllers/baseline_systems_export.go index a1317020e..a5eee0ff3 100644 --- a/manager/controllers/baseline_systems_export.go +++ b/manager/controllers/baseline_systems_export.go @@ -19,9 +19,9 @@ import ( // @Param filter[display_name] query string false "Filter" // @Param filter[os] query string false "Filter" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" diff --git a/manager/controllers/baseline_systems_export_test.go b/manager/controllers/baseline_systems_export_test.go index 04939323f..58a568567 100644 --- a/manager/controllers/baseline_systems_export_test.go +++ b/manager/controllers/baseline_systems_export_test.go @@ -112,7 +112,7 @@ func TestBaselineSystemsExportWorkloads(t *testing.T) { core.SetupTest(t) w := CreateRequestRouterWithPath( "GET", - "/1/systems?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids][in][]=ABC", + "/1/systems?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids]=ABC", nil, "application/json", BaselineSystemsExportHandler, diff --git a/manager/controllers/baselines.go b/manager/controllers/baselines.go index aae347908..8993ada09 100644 --- a/manager/controllers/baselines.go +++ b/manager/controllers/baselines.go @@ -84,18 +84,18 @@ func BaselinesListHandler(c *gin.Context) { account := c.GetInt(middlewares.KeyAccount) apiver := c.GetInt(middlewares.KeyApiver) groups := c.GetStringMapString(middlewares.KeyInventoryGroups) - filters, err := ParseInventoryFilters(c) + filters, inventoryFilters, err := ParseInventoryFilters(c, BaselineOpts) if err != nil { return } db := middlewares.DBFromContext(c) - query := buildQueryBaselines(db, filters, account, groups) + query := buildQueryBaselines(db, inventoryFilters, account, groups) if err != nil { return } // Error handled in method itself - query, meta, params, err := ListCommon(query, c, filters, BaselineOpts) + query, meta, params, err := ListCommon(query, c, filters, inventoryFilters, BaselineOpts) if err != nil { // Error handling and setting of result code & content is done in ListCommon return diff --git a/manager/controllers/filter.go b/manager/controllers/filter.go index 75bcdad8e..13c9c6f10 100644 --- a/manager/controllers/filter.go +++ b/manager/controllers/filter.go @@ -3,9 +3,10 @@ package controllers import ( "app/base/database" "fmt" + "strings" + "github.com/pkg/errors" "gorm.io/gorm" - "strings" ) type FilterData struct { @@ -16,7 +17,7 @@ type FilterData struct { type Filters map[string]FilterData // Parse a filter from field name and field value specification -func ParseFilterValue(val string) (FilterData, error) { +func ParseFilterValue(val string) FilterData { idx := strings.Index(val, ":") var operator string @@ -35,7 +36,7 @@ func ParseFilterValue(val string) (FilterData, error) { return FilterData{ Operator: operator, Values: values, - }, nil + } } func checkValueCount(operator string, nValues int) bool { @@ -88,9 +89,6 @@ func (t *FilterData) ToWhere(fieldName string, attributes database.AttrMap) (str case "leq": return fmt.Sprintf("%s <= ? ", attributes[fieldName].DataQuery), values, nil case "between": - if len(transformedValues) != 2 { - return "", []interface{}{}, errors.New("the `between` filter needs 2 values") - } return fmt.Sprintf("%s BETWEEN ? AND ? ", attributes[fieldName].DataQuery), values, nil case "in": return fmt.Sprintf("%s IN (?) ", attributes[fieldName].DataQuery), []interface{}{values}, nil @@ -151,3 +149,11 @@ func (t Filters) Apply(tx *gorm.DB, fields database.AttrMap) (*gorm.DB, error) { } return tx, nil } + +func (t Filters) Update(key string, value string) { + data := ParseFilterValue(value) + if fdata, ok := t[key]; ok { + data.Values = append(data.Values, fdata.Values...) + } + t[key] = data +} diff --git a/manager/controllers/filter_test.go b/manager/controllers/filter_test.go index c3368bb24..b4422b934 100644 --- a/manager/controllers/filter_test.go +++ b/manager/controllers/filter_test.go @@ -34,8 +34,7 @@ func TestFilterParse(t *testing.T) { } for i, f := range testFilters { - filter, err := ParseFilterValue(f) - assert.Equal(t, nil, err) + filter := ParseFilterValue(f) assert.Equal(t, operators[i], filter.Operator) assert.Equal(t, values[i], filter.Values) } @@ -52,8 +51,7 @@ func TestFilterToSql(t *testing.T) { } for i, f := range testFilters { - filter, err := ParseFilterValue(f) - assert.Equal(t, nil, err) + filter := ParseFilterValue(f) attrMap := database.AttrMap{"test": {DataQuery: "test", OrderQuery: "test", Parser: dummyParser}} query, _, err := filter.ToWhere("test", attrMap) @@ -73,8 +71,7 @@ func TestFilterToSqlAdvanced(t *testing.T) { } for i, f := range testFilters { - filter, err := ParseFilterValue(f) - assert.Equal(t, nil, err) + filter := ParseFilterValue(f) attrMap := database.AttrMap{"test": {DataQuery: "(NOT test)", OrderQuery: "(NOT test)", Parser: dummyParser}} query, _, err := filter.ToWhere("test", attrMap) assert.Equal(t, nil, err) @@ -84,8 +81,7 @@ func TestFilterToSqlAdvanced(t *testing.T) { // Filter out null characters func TestFilterInvalidValue(t *testing.T) { - filter, err := ParseFilterValue("eq:aa\u0000aa") - assert.NoError(t, err) + filter := ParseFilterValue("eq:aa\u0000aa") attrMap, _, err := database.GetQueryAttrs(struct{ V string }{""}) assert.NoError(t, err) _, value, err := filter.ToWhere("V", attrMap) diff --git a/manager/controllers/package_systems.go b/manager/controllers/package_systems.go index a9a30b3df..547fadc7f 100644 --- a/manager/controllers/package_systems.go +++ b/manager/controllers/package_systems.go @@ -105,12 +105,12 @@ func packageSystemsCommon(db *gorm.DB, c *gin.Context) (*gorm.DB, *ListMeta, []s } query := packageSystemsQuery(db, account, groups, packageName, packageIDs) - filters, err := ParseInventoryFilters(c) + filters, inventoryFilters, err := ParseInventoryFilters(c, PackageSystemsOpts) if err != nil { return nil, nil, nil, err } // Error handled in method itself - query, _ = ApplyInventoryFilter(filters, query, "sp.inventory_id") - query, meta, params, err := ListCommon(query, c, filters, PackageSystemsOpts) + query, _ = ApplyInventoryFilter(inventoryFilters, query, "sp.inventory_id") + query, meta, params, err := ListCommon(query, c, filters, inventoryFilters, PackageSystemsOpts) // Error handled in method itself return query, meta, params, err } @@ -126,9 +126,9 @@ func packageSystemsCommon(db *gorm.DB, c *gin.Context) (*gorm.DB, *ListMeta, []s // @Param offset query int false "Offset for paging" // @Param package_name path string true "Package name" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -191,9 +191,9 @@ func PackageSystemsListHandler(c *gin.Context) { // @Param offset query int false "Offset for paging" // @Param package_name path string true "Package name" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" diff --git a/manager/controllers/package_systems_export.go b/manager/controllers/package_systems_export.go index 793200dad..1896750b7 100644 --- a/manager/controllers/package_systems_export.go +++ b/manager/controllers/package_systems_export.go @@ -16,9 +16,9 @@ import ( // @Accept json // @Produce json // @Param package_name path string true "Package name" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -54,11 +54,11 @@ func PackageSystemsExportHandler(c *gin.Context) { } query := packageSystemsQuery(db, account, groups, packageName, packageIDs) - filters, err := ParseInventoryFilters(c) + _, inventoryFilters, err := ParseInventoryFilters(c, PackageSystemsOpts) if err != nil { return } // Error handled in method itself - query, _ = ApplyInventoryFilter(filters, query, "sp.inventory_id") + query, _ = ApplyInventoryFilter(inventoryFilters, query, "sp.inventory_id") query, err = ExportListCommon(query, c, PackageSystemsOpts) if err != nil { return diff --git a/manager/controllers/package_versions.go b/manager/controllers/package_versions.go index c9ef686e2..09ce3fa21 100644 --- a/manager/controllers/package_versions.go +++ b/manager/controllers/package_versions.go @@ -91,7 +91,7 @@ func PackageVersionsListHandler(c *gin.Context) { query := packageVersionsQuery(db, account, groups, packageNameIDs) // we don't support tags and filters for this endpoint - query, meta, params, err := ListCommon(query, c, nil, PackageVersionsOpts) + query, meta, params, err := ListCommon(query, c, nil, nil, PackageVersionsOpts) if err != nil { return } // Error handled in method itself diff --git a/manager/controllers/packages.go b/manager/controllers/packages.go index b66777a7d..aa44e6860 100644 --- a/manager/controllers/packages.go +++ b/manager/controllers/packages.go @@ -132,9 +132,9 @@ func packagesQuery(db *gorm.DB, filters map[string]FilterData, acc int, groups m // @Param filter[systems_applicable] query string false "Filter" // @Param filter[summary] query string false "Filter" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -150,17 +150,17 @@ func PackagesListHandler(c *gin.Context) { apiver := c.GetInt(middlewares.KeyApiver) groups := c.GetStringMapString(middlewares.KeyInventoryGroups) - filters, err := ParseInventoryFilters(c) + filters, inventoryFilters, err := ParseInventoryFilters(c, PackagesOpts) if err != nil { return } db := middlewares.DBFromContext(c) - query := packagesQuery(db, filters, account, groups) + query := packagesQuery(db, inventoryFilters, account, groups) if err != nil { return } // Error handled in method itself - query, meta, params, err := ListCommon(query, c, filters, PackagesOpts) + query, meta, params, err := ListCommon(query, c, filters, inventoryFilters, PackagesOpts) if err != nil { return } // Error handled in method itself diff --git a/manager/controllers/packages_export.go b/manager/controllers/packages_export.go index 5f942c166..13c4257d0 100644 --- a/manager/controllers/packages_export.go +++ b/manager/controllers/packages_export.go @@ -26,13 +26,13 @@ func PackagesExportHandler(c *gin.Context) { account := c.GetInt(middlewares.KeyAccount) apiver := c.GetInt(middlewares.KeyApiver) groups := c.GetStringMapString(middlewares.KeyInventoryGroups) - filters, err := ParseInventoryFilters(c) + _, inventoryfilters, err := ParseInventoryFilters(c, PackagesOpts) if err != nil { return } db := middlewares.DBFromContext(c) - query := packagesQuery(db, filters, account, groups) + query := packagesQuery(db, inventoryfilters, account, groups) if err != nil { return } diff --git a/manager/controllers/system_advisories.go b/manager/controllers/system_advisories.go index b56fc4429..12de05224 100644 --- a/manager/controllers/system_advisories.go +++ b/manager/controllers/system_advisories.go @@ -92,7 +92,7 @@ func systemAdvisoriesCommon(c *gin.Context) (*gorm.DB, *ListMeta, []string, erro } query := buildSystemAdvisoriesQuery(db, account, groups, inventoryID) - query, meta, params, err := ListCommon(query, c, nil, SystemAdvisoriesOpts) + query, meta, params, err := ListCommon(query, c, nil, nil, SystemAdvisoriesOpts) // Error handling and setting of result code & content is done in ListCommon return query, meta, params, err } diff --git a/manager/controllers/system_packages.go b/manager/controllers/system_packages.go index 95b117257..dda79ca10 100644 --- a/manager/controllers/system_packages.go +++ b/manager/controllers/system_packages.go @@ -118,11 +118,15 @@ func SystemPackagesHandler(c *gin.Context) { LogAndRespBadRequest(c, errors.New("bad request"), "incorrect inventory_id format") return } - + filters, inventoryFilters, err := ParseInventoryFilters(c, PackageSystemsOpts) + if err != nil { + // Error handled in method itself + return + } var loaded []SystemPackageDBLoad db := middlewares.DBFromContext(c) q := systemPackageQuery(db, account, groups, inventoryID) - q, meta, params, err := ListCommon(q, c, nil, SystemPackagesOpts) + q, meta, params, err := ListCommon(q, c, filters, inventoryFilters, SystemPackagesOpts) if err != nil { return } diff --git a/manager/controllers/systems.go b/manager/controllers/systems.go index 1d0b3cb57..ab9c53b5d 100644 --- a/manager/controllers/systems.go +++ b/manager/controllers/systems.go @@ -172,12 +172,12 @@ func systemsCommon(c *gin.Context, apiver int) (*gorm.DB, *ListMeta, []string, e groups := c.GetStringMapString(middlewares.KeyInventoryGroups) db := middlewares.DBFromContext(c) query := querySystems(db, account, apiver, groups) - filters, err := ParseInventoryFilters(c) + filters, inventoryFilters, err := ParseInventoryFilters(c, SystemOpts) if err != nil { return nil, nil, nil, err } // Error handled method itself - query, _ = ApplyInventoryFilter(filters, query, "sp.inventory_id") - query, meta, params, err := ListCommon(query, c, filters, SystemOpts) + query, _ = ApplyInventoryFilter(inventoryFilters, query, "sp.inventory_id") + query, meta, params, err := ListCommon(query, c, filters, inventoryFilters, SystemOpts) // Error handled method itself return query, meta, params, err } @@ -209,9 +209,9 @@ func systemsCommon(c *gin.Context, apiver int) (*gorm.DB, *ListMeta, []string, e // @Param filter[baseline_name] query string false "Filter" // @Param filter[os] query string false "Filter OS version" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -285,9 +285,9 @@ func SystemsListHandler(c *gin.Context) { // @Param filter[baseline_name] query string false "Filter" // @Param filter[os] query string false "Filter OS version" // @Param tags query []string false "Tag filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" diff --git a/manager/controllers/systems_export.go b/manager/controllers/systems_export.go index ba2a6a266..98b16fee1 100644 --- a/manager/controllers/systems_export.go +++ b/manager/controllers/systems_export.go @@ -22,9 +22,9 @@ import ( // @Param filter[other_count] query string false "Filter" // @Param filter[stale] query string false "Filter" // @Param filter[packages_installed] query string false "Filter" -// @Param filter[group_name][in] query string false "Filter systems by inventory groups" +// @Param filter[group_name] query []string false "Filter systems by inventory groups" // @Param filter[system_profile][sap_system] query string false "Filter only SAP systems" -// @Param filter[system_profile][sap_sids][in] query []string false "Filter systems by their SAP SIDs" +// @Param filter[system_profile][sap_sids] query []string false "Filter systems by their SAP SIDs" // @Param filter[system_profile][ansible] query string false "Filter systems by ansible" // @Param filter[system_profile][ansible][controller_version] query string false "Filter systems by ansible version" // @Param filter[system_profile][mssql] query string false "Filter systems by mssql version" @@ -42,11 +42,11 @@ func SystemsExportHandler(c *gin.Context) { groups := c.GetStringMapString(middlewares.KeyInventoryGroups) db := middlewares.DBFromContext(c) query := querySystems(db, account, apiver, groups) - filters, err := ParseInventoryFilters(c) + _, inventoryFilters, err := ParseInventoryFilters(c, SystemOpts) if err != nil { return } // Error handled in method itself - query, _ = ApplyInventoryFilter(filters, query, "sp.inventory_id") + query, _ = ApplyInventoryFilter(inventoryFilters, query, "sp.inventory_id") var systems SystemDBLookupSlice diff --git a/manager/controllers/systems_export_test.go b/manager/controllers/systems_export_test.go index 98e873526..6cd2bdfcb 100644 --- a/manager/controllers/systems_export_test.go +++ b/manager/controllers/systems_export_test.go @@ -101,7 +101,7 @@ func TestExportSystemsTagsInvalid(t *testing.T) { func TestSystemsExportWorkloads(t *testing.T) { w := makeRequest( t, - "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids][in][]=ABC", + "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids]=ABC", "application/json", ) diff --git a/manager/controllers/systems_ids_test.go b/manager/controllers/systems_ids_test.go index 73d13e086..bf5198dfc 100644 --- a/manager/controllers/systems_ids_test.go +++ b/manager/controllers/systems_ids_test.go @@ -74,7 +74,7 @@ func TestSystemsIDsTagsInvalid(t *testing.T) { } func TestSystemsIDsWorkloads1(t *testing.T) { - url := "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids][in][]=ABC" + url := "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids]=ABC" output := testSystemsIDs(t, url, 1) assert.Equal(t, 2, len(output.IDs)) assert.Equal(t, "00000000-0000-0000-0000-000000000001", output.IDs[0]) @@ -136,7 +136,7 @@ func TestSystemsIDsFilterBaseline(t *testing.T) { func TestSystemsIDsFilterNotExisting(t *testing.T) { statusCode, errResp := testSystemsIDsError(t, "/?filter[not-existing]=1") assert.Equal(t, http.StatusBadRequest, statusCode) - assert.Equal(t, "Invalid filter field: not-existing", errResp.Error) + assert.Equal(t, "cannot parse inventory filters: Invalid filter field: not-existing", errResp.Error) } func TestSystemsIDsFilterPartialOS(t *testing.T) { @@ -156,12 +156,6 @@ func TestSystemsIDsFilterOS(t *testing.T) { assert.Equal(t, output.Data[2].ID, outputIDs.IDs[2]) } -func TestSystemsIDsFilterInvalidSyntax(t *testing.T) { - statusCode, errResp := testSystemsIDsError(t, "/?filter[os][in]=RHEL 8.1,RHEL 7.3") - assert.Equal(t, http.StatusBadRequest, statusCode) - assert.Equal(t, InvalidNestedFilter, errResp.Error) -} - func TestSystemsIDsOrderOS(t *testing.T) { output := testSystems(t, `/?sort=os`, 1) outputIDs := testSystemsIDs(t, `/?sort=os`, 1) diff --git a/manager/controllers/systems_test.go b/manager/controllers/systems_test.go index 847ade431..d93ab8048 100644 --- a/manager/controllers/systems_test.go +++ b/manager/controllers/systems_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -const sapABCFilter = "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids][]=ABC" +const sapABCFilter = "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids]=ABC" func TestSystemsDefault(t *testing.T) { output := testSystems(t, `/`, 1) @@ -112,8 +112,28 @@ func TestSystemsTagsInvalid(t *testing.T) { assert.Equal(t, fmt.Sprintf(InvalidTagMsg, "invalidTag"), errResp.Error) } +func TestSystemsTagsEscaping1(t *testing.T) { + output := testSystems(t, `/?tags=ns1/k3=val4&tags="ns/key=quote"`, 1) + assert.Equal(t, 0, len(output.Data)) +} + +func TestSystemsTagsEscaping2(t *testing.T) { + output := testSystems(t, `/?tags=ns1/k3=val4&tags='ns/key=singlequote'`, 1) + assert.Equal(t, 0, len(output.Data)) +} + +func TestSystemsTagsEscaping3(t *testing.T) { + output := testSystems(t, `/?tags=ns1/k3=val4&tags='ns/key=inside""quote'`, 1) + assert.Equal(t, 0, len(output.Data)) +} + +func TestSystemsTagsEscaping4(t *testing.T) { + output := testSystems(t, `/?tags=ns1/k3=val4&tags=ne/key="{{malformed json}}"`, 1) + assert.Equal(t, 0, len(output.Data)) +} + func TestSystemsWorkloads1(t *testing.T) { - url := "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids][in][]=ABC" + url := "/?filter[system_profile][sap_system]=true&filter[system_profile][sap_sids]=ABC" output := testSystems(t, url, 1) assert.Equal(t, 2, len(output.Data)) assert.Equal(t, "00000000-0000-0000-0000-000000000001", output.Data[0].ID) @@ -131,6 +151,17 @@ func TestSystemsWorkloads3(t *testing.T) { assert.Equal(t, 0, len(output.Data)) } +func TestSystemsWorkloadEscaping1(t *testing.T) { + url := "/?filter[system_profile][sap_sids]='singlequote'" + output := testSystems(t, url, 1) + assert.Equal(t, 0, len(output.Data)) +} + +func TestSystemsWorkloadEscaping2(t *testing.T) { + url := `/?filter[system_profile][sap_sids]="{{malformed json}}"` + output := testSystems(t, url, 1) + assert.Equal(t, 0, len(output.Data)) +} func TestSystemsPackagesCount(t *testing.T) { output := testSystems(t, "/?sort=-packages_installed,id", 3) assert.Equal(t, 5, len(output.Data)) @@ -174,7 +205,7 @@ func TestSystemsFilterBaseline(t *testing.T) { func TestSystemsFilterNotExisting(t *testing.T) { statusCode, errResp := testSystemsError(t, "/?filter[not-existing]=1") assert.Equal(t, http.StatusBadRequest, statusCode) - assert.Equal(t, "Invalid filter field: not-existing", errResp.Error) + assert.Equal(t, "cannot parse inventory filters: Invalid filter field: not-existing", errResp.Error) } func TestSystemsFilterOS(t *testing.T) { @@ -185,12 +216,6 @@ func TestSystemsFilterOS(t *testing.T) { assert.Equal(t, "RHEL 8.1", output.Data[2].Attributes.OS) } -func TestSystemsFilterInvalidSyntax(t *testing.T) { - statusCode, errResp := testSystemsError(t, "/?filter[os][in]=RHEL 8.1,RHEL 7.3") - assert.Equal(t, http.StatusBadRequest, statusCode) - assert.Equal(t, InvalidNestedFilter, errResp.Error) -} - func TestSystemsOrderOS(t *testing.T) { output := testSystems(t, `/?sort=os`, 1) assert.Equal(t, "RHEL 7.3", output.Data[0].Attributes.OS) @@ -237,20 +262,20 @@ func TestSystemsTagsInMetadata(t *testing.T) { } func TestSAPSystemMeta1(t *testing.T) { - url := "/?filter[system_profile][sap_sids][]=ABC" + url := "/?filter[system_profile][sap_sids]=ABC" output := testSystems(t, url, 1) testMap := map[string]FilterData{ - "sap_sids": {"eq", []string{`"ABC"`}}, + "sap_sids": {"eq", []string{"ABC"}}, "stale": {"eq", []string{"false"}}, } assert.Equal(t, testMap, output.Meta.Filter) } func TestSAPSystemMeta2(t *testing.T) { - url := "/?filter[system_profile][sap_sids][in][]=ABC" + url := "/?filter[system_profile][sap_sids]=ABC" output := testSystems(t, url, 1) testMap := map[string]FilterData{ - "sap_sids": {"in", []string{`"ABC"`}}, + "sap_sids": {"eq", []string{"ABC"}}, "stale": {"eq", []string{"false"}}, } assert.Equal(t, testMap, output.Meta.Filter) @@ -261,17 +286,17 @@ func TestSAPSystemMeta3(t *testing.T) { output := testSystems(t, url, 1) testMap := map[string]FilterData{ "sap_system": {"eq", []string{"true"}}, - "sap_sids": {"eq", []string{`"ABC"`}}, + "sap_sids": {"eq", []string{"ABC"}}, "stale": {"eq", []string{"false"}}, } assert.Equal(t, testMap, output.Meta.Filter) } func TestSAPSystemMeta4(t *testing.T) { - url := "/?filter[system_profile][sap_sids][in]=ABC&filter[system_profile][sap_sids][in]=GHI" + url := "/?filter[system_profile][sap_sids]=ABC&filter[system_profile][sap_sids]=GHI" output := testSystems(t, url, 1) testMap := map[string]FilterData{ - "sap_sids": {"in", []string{`"ABC"`, `"GHI"`}}, + "sap_sids": {"eq", []string{"GHI", "ABC"}}, "stale": {"eq", []string{"false"}}, } assert.Equal(t, testMap, output.Meta.Filter) diff --git a/manager/controllers/systemtags.go b/manager/controllers/systemtags.go index 3335c92dd..ac33c38e9 100644 --- a/manager/controllers/systemtags.go +++ b/manager/controllers/systemtags.go @@ -75,7 +75,7 @@ func SystemTagListHandler(c *gin.Context) { Select(SystemTagSelect). Group("sq.tag") - tx, meta, params, err := ListCommon(query, c, nil, SystemTagsOpts) + tx, meta, params, err := ListCommon(query, c, nil, nil, SystemTagsOpts) if !checkSortMeta(meta.Sort) { LogAndRespBadRequest(c, errors.New("invalid sort field(s)"), "invalid sort") return diff --git a/manager/controllers/utils.go b/manager/controllers/utils.go index 5286c9994..8b8efff32 100644 --- a/manager/controllers/utils.go +++ b/manager/controllers/utils.go @@ -20,6 +20,7 @@ import ( ) const InvalidOffsetMsg = "Invalid offset" +const InvalidFilter = "Invalid filter field: %v" const InvalidTagMsg = "Invalid tag '%s'. Use 'namespace/key=val format'" const InvalidNestedFilter = "Nested operators not yet implemented for standard filters" const FilterNotSupportedMsg = "filtering not supported on this endpoint" @@ -37,11 +38,6 @@ var validSystemProfileFilters = map[string]bool{ "ansible->controller_version": true, } -var multiValueFilters = map[string]bool{ - "sap_sids": true, - "group_name": true, -} - func LogAndRespError(c *gin.Context, err error, respMsg string) { utils.LogError("err", err.Error(), respMsg) c.AbortWithStatusJSON(http.StatusInternalServerError, utils.ErrorResponse{Error: respMsg}) @@ -104,53 +100,50 @@ func ApplySort(c *gin.Context, tx *gorm.DB, fieldExprs database.AttrMap, return tx, appliedFields, nil } -func validateFilters(q QueryMap, allowedFields database.AttrMap) error { - for key := range q { - // system_profile and group_name are handled by tags, so it can be skipped - if key == "system_profile" || key == "group_name" { - continue - } - if _, ok := allowedFields[key]; !ok { - return errors.Errorf("Invalid filter field: %v", key) - } - } - return nil +type NestedFilterMap map[string]string + +var nestedFilters = NestedFilterMap{ + "group_name": "group_name", + "group_name][in": "group_name", // obsoleted, backward compatible + "system_profile][sap_system": "sap_system", + "system_profile][sap_sids": "sap_sids", + "system_profile][sap_sids][": "sap_sids", // obsoleted, backward compatible + "system_profile][sap_sids][in": "sap_sids", // obsoleted, backward compatible + "system_profile][sap_sids][in][": "sap_sids", // obsoleted, backward compatible + "system_profile][ansible": "ansible", + "system_profile][ansible][controller_version": "ansible->controller_version", + "system_profile][mssql": "mssql", + "system_profile][mssql][version": "mssql->version", } -func ParseFilters(q QueryMap, allowedFields database.AttrMap, - defaultFilters map[string]FilterData, apiver int) (Filters, error) { +func ParseFilters(c *gin.Context, allowedFields database.AttrMap, + defaultFilters map[string]FilterData, apiver int) (Filters, Filters, error) { filters := Filters{} - var err error + inventoryFilters := Filters{} + + params := c.Request.URL.Query() // map[string][]string + for name, values := range params { + if strings.HasPrefix(name, "filter[") { + subject := name[7 : len(name)-1] // strip key from "filter[...]" + for _, v := range values { + if _, ok := nestedFilters[subject]; ok { + nested := nestedFilters[subject] + inventoryFilters.Update(nested, v) + continue + } + if _, ok := allowedFields[subject]; !ok { + return Filters{}, Filters{}, errors.Errorf(InvalidFilter, subject) + } - if err = validateFilters(q, allowedFields); err != nil { - return filters, err + filters.Update(subject, v) + } + } } - // Apply default filters + // Apply default filters if there isn't such filter already for n, v := range defaultFilters { - filters[n] = v - } - - // nolint: scopelint - for f := range allowedFields { - if elem := q.Path(f); elem != nil { - err := elem.Visit(func(path []string, val string) error { - // If we encountered error in previous element, skip processing others - if err != nil { - return err - } - - // the filter[a][eq]=b syntax was not yet implemented - if len(path) > 0 { - err = errors.New(InvalidNestedFilter) - return err - } - filters[f], err = ParseFilterValue(val) - return err - }) - if err != nil { - return filters, err - } + if _, ok := filters[n]; !ok { + filters[n] = v } } @@ -163,7 +156,7 @@ func ParseFilters(q QueryMap, allowedFields database.AttrMap, } } - return filters, err + return filters, inventoryFilters, nil } type ListOpts struct { @@ -175,9 +168,8 @@ type ListOpts struct { } func ExportListCommon(tx *gorm.DB, c *gin.Context, opts ListOpts) (*gorm.DB, error) { - query := NestedQueryMap(c, "filter") apiver := c.GetInt(middlewares.KeyApiver) - filters, err := ParseFilters(query, opts.Fields, opts.DefaultFilters, apiver) + filters, _, err := ParseFilters(c, opts.Fields, opts.DefaultFilters, apiver) if err != nil { LogAndRespBadRequest(c, err, err.Error()) return nil, errors.Wrap(err, "filters parsing failed") @@ -202,10 +194,9 @@ func extractTagsQueryString(c *gin.Context) string { } // nolint: funlen, lll -func ListCommon(tx *gorm.DB, c *gin.Context, tagFilter map[string]FilterData, opts ListOpts, params ...string) ( +func ListCommon(tx *gorm.DB, c *gin.Context, filters Filters, tagFilter Filters, opts ListOpts, params ...string) ( *gorm.DB, *ListMeta, []string, error) { hasSystems := true - apiver := c.GetInt(middlewares.KeyApiver) limit, offset, err := utils.LoadLimitOffset(c, core.DefaultLimit) if err != nil { LogAndRespBadRequest(c, err, err.Error()) @@ -213,9 +204,6 @@ func ListCommon(tx *gorm.DB, c *gin.Context, tagFilter map[string]FilterData, op } tx, searchQ := ApplySearch(c, tx, opts.SearchFields...) - query := NestedQueryMap(c, "filter") - - filters, err := ParseFilters(query, opts.Fields, opts.DefaultFilters, apiver) if err != nil { LogAndRespBadRequest(c, err, err.Error()) return nil, nil, nil, errors.Wrap(err, "filters parsing failed") @@ -298,39 +286,16 @@ func ApplySearch(c *gin.Context, tx *gorm.DB, searchColumns ...string) (*gorm.DB } type Tag struct { - Namespace *string - Key string - Value *string + Namespace *string `json:"namespace,omitempty"` + Key string `json:"key"` + Value *string `json:"value,omitempty"` } -func HasInventoryFilter(c *gin.Context) bool { +func HasInventoryFilter(filters Filters) bool { if !enableCyndiTags { return false } - hasFilter := false - if len(c.QueryArray("tags")) > 0 { - hasFilter = true - } - - // If we have the `system_profile` filter item, then we have tags - spQuery := NestedQueryMap(c, "filter").Path("system_profile") - if spQuery != nil { - // nolint: errcheck - spQuery.Visit(func(path []string, val string) error { - hasFilter = true - return nil - }) - } - - groupsQuery := NestedQueryMap(c, "filter").Path("group_name") - if groupsQuery != nil { - // nolint: errcheck - groupsQuery.Visit(func(path []string, val string) error { - hasFilter = true - return nil - }) - } - return hasFilter + return len(filters) > 0 } func trimQuotes(s string) string { @@ -373,33 +338,28 @@ func (t *Tag) ApplyTag(tx *gorm.DB) *gorm.DB { return tx } - ns := "" - if t.Namespace != nil { - ns = fmt.Sprintf(`"namespace": "%s",`, *t.Namespace) - } - - v := "" - if t.Value != nil { - v = fmt.Sprintf(`, "value":"%s"`, *t.Value) - } - - query := fmt.Sprintf(`[{%s "key": "%s" %s}]`, ns, t.Key, v) - return tx.Where("ih.tags @> ?::jsonb", query) + tagStr, _ := json.Marshal([]Tag{*t}) + return tx.Where("ih.tags @> ?::jsonb", tagStr) } -func ParseInventoryFilters(c *gin.Context) (map[string]FilterData, error) { - filters := Filters{} +func ParseInventoryFilters(c *gin.Context, opts ListOpts) (Filters, Filters, error) { + tagFilters := Filters{} - err := parseTagsFromCtx(c, filters) + err := parseTagsFromCtx(c, tagFilters) if err != nil { - return nil, err + return nil, nil, err } - if err := parseFiltersFromCtx(c, filters); err != nil { - return nil, errors.Wrap(err, "cannot parse inventory filters") + apiver := c.GetInt(middlewares.KeyApiver) + filters, inventoryFilters, err := ParseFilters(c, opts.Fields, opts.DefaultFilters, apiver) + if err != nil { + err = errors.Wrap(err, "cannot parse inventory filters") + LogAndRespBadRequest(c, err, err.Error()) + return nil, nil, err } - return filters, nil + mergeMaps(tagFilters, inventoryFilters) + return filters, tagFilters, nil } func parseTagsFromCtx(c *gin.Context, filters Filters) error { @@ -434,71 +394,6 @@ func parseTagsFromCtx(c *gin.Context, filters Filters) error { return nil } -func parseFiltersFromCtx(c *gin.Context, filters Filters) error { - var filterItems []QueryItem - filter := NestedQueryMap(c, "filter") - - if item, ok := filter.GetPath("system_profile"); ok { - // filter nested under "system_profile" - filterItems = append(filterItems, item) - } - if _, ok := filter.GetPath("group_name"); ok { - // "group_name" filter - filterItems = append(filterItems, filter) - } - - for _, filter := range filterItems { - err := filter.Visit(func(path []string, val string) error { - // Specific filter keys - if len(path) >= 1 && multiValueFilters[path[0]] { - return visitMultiValueFilters(filters, path, val) - } - - // [group_name] filter can be processed only with Specific filter keys - if f, ok := filter.(QueryMap); ok { - if _, has := f.GetPath("group_name"); has { - return nil - } - } - - // Generic [system_profile] filter keys - return visitSystemProfileFilters(filters, path, val) - }) - if err != nil { - LogAndRespBadRequest(c, err, err.Error()) - return err - } - } - return nil -} - -func visitMultiValueFilters(filters Filters, path []string, val string) error { - val = strconv.Quote(val) - var op string - if op = "eq"; len(path) > 1 { - op = path[1] - } - err := appendFilterData(filters, path[0], op, val) - if err != nil { - return errors.Wrapf(err, "invalid filter value: %s", val) - } - return nil -} - -// Builds key in following format path[0]->path[1]->path[2]... -func visitSystemProfileFilters(filters Filters, path []string, val string) error { - var key string - for i, s := range path { - if i == 0 { - key = path[0] - continue - } - key = fmt.Sprintf("%s->%s", key, s) - } - err := appendFilterData(filters, key, "eq", val) - return errors.Wrapf(err, "invalid filter value: %s", val) -} - // Filter systems by tags with subquery func ApplyInventoryFilter(filters map[string]FilterData, tx *gorm.DB, systemIDExpr string) (*gorm.DB, bool) { if !enableCyndiTags { @@ -531,19 +426,7 @@ func ApplyInventoryWhere(filters map[string]FilterData, tx *gorm.DB) (*gorm.DB, } if validSystemProfileFilters[key] { - values := strings.Join(val.Values, ",") - q := buildSystemProfileQuery(key, values) - - // Builds array of values - if len(val.Values) > 1 { - values = fmt.Sprintf("[%s]", values) - } - - if values == "not_nil" { - tx = tx.Where(q) - } else { - tx = tx.Where(q, values) - } + tx = buildSystemProfileQuery(tx, key, val.Values) applied = true continue @@ -552,10 +435,7 @@ func ApplyInventoryWhere(filters map[string]FilterData, tx *gorm.DB) (*gorm.DB, if strings.Contains(key, "group_name") { groups := []string{} for _, v := range val.Values { - name, err := strconv.Unquote(v) - if err != nil { - name = v - } + name := v group, err := utils.ParseInventoryGroup(nil, &name) if err != nil { // couldn't marshal inventory group to json @@ -575,14 +455,18 @@ func ApplyInventoryWhere(filters map[string]FilterData, tx *gorm.DB) (*gorm.DB, // Example: // buildSystemProfileQuery("mssql->version", "1.0") // returns "(ih.system_profile -> 'mssql' ->> 'version')::text = 1.0" -func buildSystemProfileQuery(key string, val string) string { +func buildSystemProfileQuery(tx *gorm.DB, key string, values []string) *gorm.DB { var cmp string + var val string switch key { case "sap_sids": cmp = "::jsonb @> ?::jsonb" + bval, _ := json.Marshal(values) + val = string(bval) default: cmp = "::text = ?" + val = values[0] } if val == "not_nil" { @@ -600,120 +484,12 @@ func buildSystemProfileQuery(key string, val string) string { } } - return fmt.Sprintf("%s%s", subq, cmp) -} - -type QueryItem interface { - IsQuery() - Visit(visitor func(path []string, val string) error, pathPrefix ...string) error -} -type QueryArr []string - -func (QueryArr) IsQuery() {} - -func (q QueryArr) Visit(visitor func(path []string, val string) error, pathPrefix ...string) error { - for _, item := range q { - if err := visitor(pathPrefix, item); err != nil { - return err - } - } - return nil -} - -type QueryMap map[string]QueryItem - -func (QueryMap) IsQuery() {} - -func (q QueryMap) Visit(visitor func(path []string, val string) error, pathPrefix ...string) error { - for k, v := range q { - newPath := make([]string, len(pathPrefix)) - copy(newPath, pathPrefix) - newPath = append(newPath, k) - switch val := v.(type) { - case QueryMap: - if err := val.Visit(visitor, newPath...); err != nil { - return err - } - case QueryArr: - // Inlined code here to avoid calling interface method in struct receiver (low perf) - for _, item := range val { - if err := visitor(newPath, item); err != nil { - return err - } - } - } - } - return nil -} - -func (q QueryMap) GetPath(keys ...string) (QueryItem, bool) { - var item QueryItem - item, has := q, true - - for has && item != nil && len(keys) > 0 { - switch itemMap := item.(type) { - case QueryMap: - item, has = itemMap[keys[0]] - keys = keys[1:] - - default: - break - } - } - return item, has -} - -func (q QueryMap) Path(keys ...string) QueryItem { - v, _ := q.GetPath(keys...) - return v -} - -func NestedQueryMap(c *gin.Context, key string) QueryMap { - return nestedQueryImpl(c.Request.URL.Query(), key) -} - -func (q *QueryMap) appendValue(steps []string, value []string) { - res := *q - for i, v := range steps { - if i == len(steps)-1 { - res[v] = QueryArr(value) - } else { - if _, has := res[v]; !has { - res[v] = QueryMap{} - } - res = res[v].(QueryMap) - } + subq = fmt.Sprintf("%s%s", subq, cmp) + if val == "not_nil" { + return tx.Where(subq) } -} -// nolint: gocognit -func nestedQueryImpl(values map[string][]string, key string) QueryMap { - root := QueryMap{} - - for name, value := range values { - var steps []string - var i int - var j int - for len(name) > 0 && i >= 0 && j >= 0 { - if i = strings.IndexByte(name, '['); i >= 0 { - if name[0:i] == key || len(steps) > 0 { - // if j is 0 here, that means we received []as a part of query param name, should indicate an array - if j = strings.IndexByte(name[i+1:], ']'); j >= 0 { - // Skip [] in param names - if len(name[i+1:][:j]) > 0 { - steps = append(steps, name[i+1:][:j]) - } - name = name[i+j+2:] - } - } else if name[0:i] != key && steps == nil { - // Invalid key for the context - abort. - return root - } - } - } - root.appendValue(steps, value) - } - return root + return tx.Where(subq, val) } func Csv(ctx *gin.Context, code int, res interface{}) { @@ -924,35 +700,6 @@ func mergeMaps(first map[string]FilterData, second map[string]FilterData) { } } -func appendFilterData(filters Filters, key string, op, val string) error { - if _, ok := filters[key]; !ok { - filters[key] = FilterData{} - } - - values, err := splitValues(op, val) - if err != nil { - return err - } - - filters[key] = FilterData{ - Operator: op, - Values: append(filters[key].Values, values...), - } - return nil -} - -// Split comma delimited string values for IN operator -func splitValues(op, val string) ([]string, error) { - if op == "in" && strings.Contains(val, ",") { - val, err := strconv.Unquote(val) - if err != nil { - return []string{}, errors.Wrap(err, "cannot unquote value") - } - return strings.Split(val, ","), nil - } - return []string{val}, nil -} - // Pagination query for handlers where ListCommon is not used func Paginate(tx *gorm.DB, limit *int, offset *int) (int, int, error) { var total int64 diff --git a/manager/controllers/utils_test.go b/manager/controllers/utils_test.go index 5bb0c0342..cd5254bd5 100644 --- a/manager/controllers/utils_test.go +++ b/manager/controllers/utils_test.go @@ -6,65 +6,20 @@ import ( "app/base/utils" "net/http" "net/http/httptest" - "reflect" "testing" - "time" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" ) -func TestNestedQueryParse(t *testing.T) { - q1 := map[string][]string{ - "filter[abc][efg][eq]": {"a"}, - "filter[a][]": {"b", "c"}, - // Check that we stop after we encountered invalid filter syntax - "filter[]]]]]": {}, - "filter[[[[[": {}, - "filter": {}, - } - res := nestedQueryImpl(q1, "filter") - // nolint: errcheck - res.Visit(func(keys []string, val string) error { - if reflect.DeepEqual([]string{"abc", "efg", "eq"}, keys) { - assert.Equal(t, "a", val) - } - // We need to be able to parse multi-value elems - if reflect.DeepEqual([]string{"a"}, keys) { - assert.Contains(t, []string{"b", "c"}, val) - } - return nil - }) -} - -func TestNestedQueryInvalidKey(t *testing.T) { - timeout := time.After(5 * time.Second) - done := make(chan bool) - - go func() { - q := map[string][]string{ - "filter[abc][efg][eq]": {"a"}, - } - res := nestedQueryImpl(q, "filte") - assert.Equal(t, res, QueryMap{}) - done <- true - }() - - select { - case <-timeout: - t.Fatal("Timeout exceeded - probably infinite loop in nested query") - case <-done: - } -} - func TestGroupNameFilter(t *testing.T) { utils.SkipWithoutDB(t) database.Configure() c, _ := gin.CreateTestContext(httptest.NewRecorder()) - c.Request, _ = http.NewRequest("GET", "/?filter[group_name][in]=group2", nil) + c.Request, _ = http.NewRequest("GET", "/?filter[group_name]=group2", nil) - filters, err := ParseInventoryFilters(c) + _, inventoryFilters, err := ParseInventoryFilters(c, ListOpts{}) assert.Nil(t, err) var systems []SystemsID @@ -72,7 +27,7 @@ func TestGroupNameFilter(t *testing.T) { rbac.KeyGrouped: `{"[{\"id\":\"inventory-group-1\"}]","[{\"id\":\"inventory-group-2\"}]"}`, } tx := database.Systems(database.Db, 1, groups) - tx, _ = ApplyInventoryFilter(filters, tx, "sp.inventory_id") + tx, _ = ApplyInventoryFilter(inventoryFilters, tx, "sp.inventory_id") tx.Scan(&systems) assert.Equal(t, 2, len(systems)) // 2 systems with `group2` in test_data @@ -85,9 +40,9 @@ func TestGroupNameFilter2(t *testing.T) { database.Configure() c, _ := gin.CreateTestContext(httptest.NewRecorder()) - c.Request, _ = http.NewRequest("GET", "/?filter[group_name][in]=group1,group2", nil) + c.Request, _ = http.NewRequest("GET", "/?filter[group_name]=group1,group2", nil) - filters, err := ParseInventoryFilters(c) + _, inventoryFilters, err := ParseInventoryFilters(c, ListOpts{}) assert.Nil(t, err) var systems []SystemsID @@ -95,7 +50,7 @@ func TestGroupNameFilter2(t *testing.T) { rbac.KeyGrouped: `{"[{\"id\":\"inventory-group-1\"}]","[{\"id\":\"inventory-group-2\"}]"}`, } tx := database.Systems(database.Db, 1, groups) - tx, _ = ApplyInventoryFilter(filters, tx, "sp.inventory_id") + tx, _ = ApplyInventoryFilter(inventoryFilters, tx, "sp.inventory_id") tx.Scan(&systems) assert.Equal(t, 8, len(systems)) // 2 systems with `group2`, 6 with `group1` in test_data