[qualys_gav] Fix rate limit headers missing error

packages/qualys_gav/_dev/deploy/docker/files/config.yml

@@ -6,10 +6,6 @@ rules:
         headers:
           Content-Type:
             - 'application/json'
-          x-ratelimit-limit: ["100"]
-          x-ratelimit-remaining: ["99"]
-          x-ratelimit-window-sec: ["3600"]
-          x-ratelimit-towait-sec: ["0"]
         body: "xxxx"
   - path: /rest/2.0/search/am/asset
     methods: ['POST']

packages/qualys_gav/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.1"
+  changes:
+    - description: Fix rate limit headers missing error.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/16810
 - version: "0.7.0"
   changes:
     - description: Enhance the data collection of the `asset` data stream to generate unique identifiers for each interval of ingestion.

packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs

@@ -70,42 +70,6 @@ program: |
             },
             "want_more": false,
           }
-        ).with(
-          resp.Header.transformMapEntry(k, v,
-            // Canonicalise header keys to match rate_limit conventions.
-            // -Limit, -Remaining and -Reset are magic suffixes in rate_limit.
-            {
-              k.has_suffix("-Limit") ?
-                (k.trim_suffix("-Limit").to_lower() + "-Limit")
-              : k.has_suffix("-Remaining") ?
-                (k.trim_suffix("-Remaining").to_lower() + "-Remaining")
-              :
-                k.to_lower(): v,
-            }
-          ).as(headers,
-            // Calculate rate limits.
-            rate_limit(
-              headers.with(
-                {
-                  "x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))],
-                }
-              ),
-              "x-ratelimit",
-              false,
-              true,
-              duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"),
-              0
-            )
-          ).as(rate_headers, rate_headers.with({
-            // Work around inf detection in input.
-            // If the headers are missing or rate_limit failed, rate and
-            // next may be missing. So use optional types.
-            ?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
-            ?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
-          })).as(limit, {
-            "header": resp.Header,
-            "rate_limit": limit,
-          })
         ))
       ).as(token,
         has(token.events) ? token : // Exit early due to failure.

packages/qualys_gav/data_stream/asset/sample_event.json

@@ -1,9 +1,9 @@
 {
-    "@timestamp": "2025-12-19T06:43:54.295Z",
+    "@timestamp": "2026-01-06T05:40:56.128Z",
     "agent": {
-        "ephemeral_id": "3d2f2b20-f2f5-4a59-af8b-3e033e6ad2cf",
-        "id": "28d54cd8-504f-4cc5-82ea-ead9cbeeb1d9",
-        "name": "elastic-agent-87299",
+        "ephemeral_id": "ffa707da-f46d-4e4e-a907-410225d59911",
+        "id": "40b6ff88-a87e-49d3-8f24-7d06fc276d40",
+        "name": "elastic-agent-25134",
         "type": "filebeat",
         "version": "8.19.2"
     },
@@ -27,7 +27,7 @@
     },
     "data_stream": {
         "dataset": "qualys_gav.asset",
-        "namespace": "92896",
+        "namespace": "15389",
         "type": "logs"
     },
     "device": {
@@ -40,7 +40,7 @@
         "version": "8.17.0"
     },
     "elastic_agent": {
-        "id": "28d54cd8-504f-4cc5-82ea-ead9cbeeb1d9",
+        "id": "40b6ff88-a87e-49d3-8f24-7d06fc276d40",
         "snapshot": false,
         "version": "8.19.2"
     },
@@ -51,7 +51,7 @@
         ],
         "created": "2025-07-09T14:21:12.000Z",
         "dataset": "qualys_gav.asset",
-        "ingested": "2025-12-19T06:43:57Z",
+        "ingested": "2026-01-06T05:40:59Z",
         "kind": "event",
         "original": "{\"activity\":{\"lastScannedDate\":1752243670000,\"source\":\"EASM\"},\"address\":\"216.160.83.56\",\"agent\":{\"activations\":[{\"key\":\"httpd\",\"status\":\"ACTIVE\"}],\"configurationProfile\":\"Apache HTTP Server\",\"connectedFrom\":\"216.160.83.56\",\"errorStatus\":false,\"lastActivity\":1752520814000,\"lastCheckedIn\":1752520814000,\"lastInventory\":1752520816000,\"udcManifestAssigned\":false,\"version\":\"2.4.7\"},\"agentId\":\"bda51f1d-13cf-49ad-a3a0-9f83debbe5a9\",\"asn\":\"AS53831\",\"assetId\":67533741,\"assetName\":\"test_asset\",\"assetType\":\"HOST\",\"assetUUID\":\"bda51f1d-13cf-49ad-a3a0-9f83debbe5a9\",\"assignedLocation\":{\"city\":\"Pune\",\"country\":\"IN\",\"name\":\"4492 Camino De La Plaza, Pune,IN\",\"state\":\"MH\"},\"biosAssetTag\":\"Test asset tag\",\"biosDescription\":\"Test\",\"biosSerialNumber\":\"Test serial number\",\"businessAppListData\":{\"businessApp\":[{\"businessCriticality\":\"2 - Less Critical\",\"environment\":\"Production\",\"id\":\"BARCODE283904\",\"managedBy\":\"user\",\"name\":\"Quoting App\",\"operationalStatus\":\"Mended\",\"ownedBy\":\"ownerr\",\"status\":\"Installed\",\"supportGroup\":\"SME Operations\",\"supportedBy\":\"sopporter\",\"usedFor\":\"Production\"}]},\"businessInformation\":{\"company\":\"Qualys\",\"department\":\"Engineering\",\"environment\":\"QA\",\"managedBy\":\"Amit\",\"operationalStatus\":\"Blocked\",\"ownedBy\":\"Paul\",\"supportGroup\":\"ABC_01\",\"supportedBy\":\"Nick\"},\"cloudProvider\":{\"alibaba\":null,\"aws\":{\"ec2\":{\"accountId\":\"1234\",\"availabilityZone\":\"us-west-2a\",\"hasAgent\":true,\"hostname\":\"hostname_value\",\"imageId\":\"imageId_value\",\"instanceId\":\"instanceId_value\",\"instanceState\":\"RUNNING\",\"instanceType\":\"m4.large\",\"kernelId\":null,\"launchdate\":1653386892000,\"privateDNS\":\"privateDNS_value\",\"privateIpAddress\":\"10.0.0.1\",\"publicDNS\":null,\"publicIpAddress\":\"175.16.199.1\",\"qualysScanner\":false,\"region\":{\"code\":\"us-west-2\",\"name\":\"US West (Oregon)\"},\"spotInstance\":false,\"subnetId\":\"subnetId_value\",\"vpcId\":\"vpcId_value\"},\"tags\":[{\"key\":\"tags_key_1\",\"value\":\"tags_value_1\"},{\"key\":\"tags_key_2\",\"value\":\"tags_value_2\"}]},\"azure\":null,\"gcp\":null,\"ibm\":null,\"oci\":null},\"container\":{\"hasSensor\":\"temp_value\",\"noOfContainers\":5,\"noOfImages\":3,\"product\":\"mock_product\",\"version\":\"mock_version\"},\"cpuCount\":0,\"createdDate\":\"2025-07-09T14:21:12.000Z\",\"criticality\":{\"isDefault\":false,\"lastUpdated\":\"2025-07-09T14:21:11.000Z\",\"score\":3},\"customAttributes\":[{\"connectorName\":\"Qualys\",\"key\":\"Media State4\",\"value\":\"Media disconnected\"}],\"dnsName\":\"test_dns\",\"domain\":[\"domain1\",\"domain2\"],\"domainRole\":\"Member Workstation\",\"easmTags\":[\"cloud\",\"cdn\"],\"hardware\":{\"category\":\"Mock category 1 / Mock category 2\",\"category1\":\"Mock category 1\",\"category2\":\"Mock category 2\",\"fullName\":\"Mock hardware\",\"lifecycle\":{\"eosDate\":\"2025-07-09T14:21:12.000Z\",\"gaDate\":\"2025-07-09T14:21:12.000Z\",\"introDate\":\"2025-07-09T14:21:12.000Z\",\"lifeCycleConfidence\":\"Exact\",\"obsoleteDate\":\"2025-07-09T14:21:12.000Z\",\"stage\":\"Not Applicable\"},\"manufacturer\":\"Mock manufacturer\",\"model\":\"Mock model\",\"productFamily\":\"Mock product family\",\"productName\":\"Mock product name\",\"productUrl\":\"https://mock_product_url.com\",\"taxonomy\":{\"category1\":\"Mock category 1\",\"category2\":\"Mock category 2\",\"id\":\"mock_hardware_taxonomy_id\",\"name\":\"Mock hardware taxonomy name\"}},\"hostId\":1437386,\"hostingCategory1\":\"CDN\",\"hwUUID\":\"422a2b16-4c8b-588a-a20c-c1851ad7e376\",\"inventory\":{\"created\":1752070872000,\"lastUpdated\":1752243670000,\"source\":\"EASM\"},\"inventory_list_data\":{\"inventory\":[{\"created\":\"2024-01-12T03:40:12.000Z\",\"last_updated\":\"2025-11-04T08:08:04.000Z\",\"source\":\"EASM\"}]},\"isContainerHost\":false,\"isp\":\"test, Inc.\",\"lastBoot\":\"2025-07-09T14:21:12.000Z\",\"lastLocation\":{\"city\":\"New York\",\"continent\":\"North America\",\"country\":\"United States\",\"name\":\"United States\",\"postal\":\"94040\",\"state\":\"California\"},\"lastLoggedOnUser\":\"test_user\",\"lastModifiedDate\":\"2025-07-11T14:21:10.000Z\",\"lparId\":\"mock_lpar_id\",\"missingSoftware\":[\"test1\",\"test2\",\"test3\"],\"netbiosName\":\"test_bios\",\"networkInterfaceListData\":{\"networkInterface\":[{\"addressIpV4\":\"81.2.69.142\",\"addressIpV6\":\"::ffff:5102:458e\",\"addresses\":\"mock_Address\",\"dnsAddress\":\"mock_dns_address\",\"gatewayAddress\":\"mock_geteaway_address\",\"hostname\":\"mock_hostname\",\"interfaceName\":\"mock_interface_name\",\"macAddress\":\"00:0c:29:15:6a:72\",\"macVendorIntroDate\":946944000000,\"manufacturer\":\"Mock manufacturer\",\"netmask\":\"mock_net_mask\"}]},\"openPortListData\":{\"openPort\":[{\"authorization\":\"Mock authorization\",\"description\":\"http protocol over TLS/SSL\",\"detectedService\":\"HTTPs\",\"detectionScore\":100,\"discoverySources\":\"EASM\",\"firstFound\":\"2025-07-09T14:21:12.000Z\",\"lastUpdated\":\"2025-07-09T14:21:12.000Z\",\"port\":443,\"protocol\":\"TCP\"}]},\"operatingSystem\":{\"architecture\":\"x86\",\"category\":\"Operating System / Windows\",\"category1\":\"Windows\",\"category2\":\"Windows\",\"cpe\":\"mock_cpe\",\"cpeId\":\"mock_cpe_id\",\"cpeType\":\"Mock cpe type\",\"edition\":\"Enterprise\",\"fullName\":\"Microsoft Windows 10 Enterprise\",\"installDate\":\"2025-07-09T14:21:12.000Z\",\"lifecycle\":{\"detectionScore\":100,\"eolDate\":\"2025-07-09T14:21:12.000Z\",\"eolSupportStage\":\"End-of-life\",\"eosDate\":\"2025-07-09T14:21:12.000Z\",\"eosSupportStage\":\"End-of-life\",\"gaDate\":\"2025-07-09T14:21:12.000Z\",\"lifeCycleConfidence\":\"Exact\",\"stage\":\"End-of-life\"},\"marketVersion\":\"10.0.19042.1052\",\"osName\":\"Windows 10\",\"productFamily\":\"Mock product family\",\"productName\":\"Microsoft Windows 10 Enterprise\",\"productUrl\":\"https://mock_product_url.com\",\"publisher\":\"test\",\"release\":\"Mock release\",\"taxonomy\":{\"category1\":\"Mock category1\",\"category2\":\"Mock category2\",\"id\":\"mock_taxonomy_id\",\"name\":\"Mock taxonomy name\"},\"update\":\"22.04 LTS 22.04.5 LTS\",\"version\":\"10.0.19042.1052\"},\"organizationName\":[\"mock\"],\"processor\":{\"coresPerSocket\":2,\"description\":\"Intel(R) Xeon(R) Gold 6430\",\"multithreadingStatus\":\"test\",\"noOfSocket\":2,\"numCPUs\":4,\"speed\":3200,\"threadsPerCore\":2},\"provider\":\"AWS\",\"riskScore\":0,\"sensor\":{\"activatedForModules\":[\"mock_activated_module\"],\"firstEasmScanDate\":1752243670000,\"lastComplianceScan\":0,\"lastEasmScanDate\":1752243670000,\"lastFullScan\":0,\"lastPcScanDateAgent\":0,\"lastPcScanDateScanner\":0,\"lastVMScan\":0,\"lastVmScanDateAgent\":0,\"lastVmScanDateScanner\":0,\"pendingActivationForModules\":[\"mock_pending_module\"]},\"sensorLastUpdatedDate\":\"2025-07-11T14:21:10.000Z\",\"serviceList\":{\"service\":[{\"description\":\"temp_Decp\",\"name\":\"systemd-networkd.service\",\"status\":\"loaded/active/running\"}]},\"softwareComponent\":\"Apache HTTP Server\",\"softwareListData\":{\"software\":[{\"architecture\":\"x86_64\",\"authorization\":\"Mock authorization\",\"authorizationDetectionScore\":5,\"category\":\"Network Application / Web Servers\",\"category1\":\"Network Application\",\"category2\":\"Web Servers\",\"component\":\"Server\",\"cpe\":\"mock_cpe\",\"cpeId\":\"mock_cpe_id\",\"cpeType\":\"Mock cpe type\",\"discoveredName\":\"Mock discovered name\",\"discoveredPublisher\":\"Mock discovered publisher\",\"discoveredVersion\":\"mock_version\",\"discoverySources\":\"EASM\",\"edition\":\"Unknown\",\"formerlyKnownAs\":\"httpd\",\"fullName\":\"Apache HTTP Server\",\"id\":8464359598295418000,\"ignoredReason\":\"Insufficient Information\",\"installDate\":\"2021-10-25T14:21:12.000Z\",\"installPath\":\"/usr/local/apache2\",\"isIgnored\":false,\"isPackage\":false,\"isPackageComponent\":false,\"language\":\"C\",\"lastUpdated\":\"2021-10-25T14:21:12.000Z\",\"lastUseDate\":\"2021-10-25T14:21:12.000Z\",\"license\":{\"category\":\"Mock license category\",\"subcategory\":\"Mock license subcategory\"},\"lifecycle\":{\"detectionScore\":0,\"eolDate\":\"2021-10-25T14:21:12.000Z\",\"eolSupportStage\":\"Mock eol support stage\",\"eosDate\":\"2021-10-25T14:21:12.000Z\",\"eosSupportStage\":\"Mock eos support stage\",\"gaDate\":\"2021-10-25T14:21:12.000Z\",\"lifeCycleConfidence\":\"Exact\",\"stage\":\"Not Applicable\"},\"marketVersion\":\"Unknown\",\"packageName\":null,\"productName\":\"Apache HTTP Server\",\"productUrl\":\"https://en.wikipedia.org/wiki/Apache_HTTP_Server,,\",\"publisher\":\"Apache\",\"softwareInstances\":[{\"BIN_PATH\":\"/usr/bin/docker -H unix:///var/run/docker.sock\",\"CONF_PATH\":\"/etc/docker/daemon.json\",\"InstanceName\":\"DOCKER\",\"PROC\":\"   1487       1 root     /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock\",\"PRODUCT\":\"Docker\",\"TECHNOLOGY\":\"Docker CE_EE\",\"VERSION\":\"24.0.7\",\"firstSeen\":\"2025-07-08T01:15:52.000Z\",\"lastSeen\":\"2025-07-14T19:20:15.000Z\"}],\"softwareType\":\"Application\",\"supportStageDesc\":\"Mock support stage desc\",\"update\":\"2021-10-25\",\"version\":\"2.4.7\"}]},\"subdomain\":[\"subdomain1\",\"subdomain2\"],\"tagList\":{\"tag\":[{\"backgroundColor\":0,\"businessImpact\":\"mock_business_impact\",\"criticalityScore\":3,\"foregroundColor\":0,\"tagId\":25971788,\"tagName\":\"Shodan\"}]},\"timeZone\":\"+05:30\",\"totalMemory\":10,\"userAccountListData\":{\"userAccount\":[{\"name\":\"root\"},{\"name\":\"serviceuser\"},{\"name\":\"devuser\"}]},\"volumeListData\":{\"volume\":[{\"free\":34645118976,\"name\":\"/\",\"size\":48202350592}]},\"whois\":[{\"createdDate\":\"2024-02-23T00:00:00.000Z\",\"dnssec\":\"test\",\"domain\":\"test_domainr\",\"domainStatus\":\"clientDeleteProhibited clientRenewProhibited clientTransferProhibited clientUpdateProhibited\",\"expirationDate\":\"2026-02-23T00:00:00.000Z\",\"registrantContact\":\"temp\",\"registrantCountry\":\"UNITED STATES\",\"registrantEmail\":\"[email protected]\",\"registrantName\":\"1API GmbH\",\"registrantOrganization\":\"Domains By Proxy, LLC\",\"registrar\":\"1API GmbH\",\"updatedDate\":\"2025-07-13T00:00:00.000Z\"}]}",
         "risk_score": 0,
@@ -284,8 +284,8 @@
             "host_id": "1437386",
             "hosting_category1": "CDN",
             "hw_uuid": "422a2b16-4c8b-588a-a20c-c1851ad7e376",
-            "interval_id": "bccc2be9-b8f0-4491-b942-26524b3ac346",
-            "interval_start": "2025-12-19T06:43:54.293Z",
+            "interval_id": "312d9394-b43b-4a60-b1e3-679e6fe993c5",
+            "interval_start": "2026-01-06T05:40:56.125Z",
             "inventory": {
                 "created": "2025-07-09T14:21:12.000Z",
                 "last_updated": "2025-07-11T14:21:10.000Z",