Skip to content

Commit 272aaee

Browse files
friedrichgfriedrichg
authored
Reorganize limits better and allow to use limits.cpu and limits.memory too (#37)
Signed-off-by: Friedrich Gonzalez <[email protected]>
1 parent 006c8fb commit 272aaee

15 files changed

+41
-65
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* [CHANGE] Add default instance max inflight pushes for distributors
66
* [CHANGE] Remove mem-ballast from distributor and querier.
77
* [CHANGE] Increase cpu requests for querier to 2.
8-
* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on requested cpu and memory
8+
* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on cpu and memory requests or limits
99
* [CHANGE] Add default tenant shard sizes
1010
* [CHANGE] Use cortex v1.15.3
1111
* [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility

‎cortex/alertmanager.libsonnet‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,6 @@
9999
container.withPorts($.util.defaultPorts + mode.ports) +
100100
container.withEnvMap($.alertmanager_env_map) +
101101
container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
102-
container.withEnvMixin([
103-
envType.withName('GOMAXPROCS') +
104-
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
105-
envType.withName('GOMEMLIMIT') +
106-
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
107-
]) +
108102
container.withArgsMixin(
109103
$.util.mapToFlags($.alertmanager_args) +
110104
mode.flags
@@ -117,6 +111,7 @@
117111
) +
118112
$.util.resourcesRequests('100m', '1Gi') +
119113
$.util.readinessProbe +
114+
$.go_container_mixin +
120115
$.jaeger_mixin
121116
else {},
122117

‎cortex/compactor.libsonnet‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,17 +45,12 @@
4545
container.withPorts($.compactor_ports) +
4646
container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
4747
container.withEnvMap($.compactor_env_map) +
48-
container.withEnvMixin([
49-
envType.withName('GOMAXPROCS') +
50-
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
51-
envType.withName('GOMEMLIMIT') +
52-
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
53-
]) +
5448
container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
5549
// Do not limit compactor CPU and request enough cores to honor configured max concurrency.
5650
$.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '5Gi') +
5751
$.util.resourcesLimits(null, '6Gi') +
5852
$.util.readinessProbe +
53+
$.go_container_mixin +
5954
$.jaeger_mixin,
6055

6156
compactor_env_map:: {

‎cortex/config.libsonnet‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,8 +401,35 @@
401401
max_series: 4.8e+6, // Max number of series per ingester. 0 = no limit. 4.8 million is closely tied to 15Gb in requests per ingester
402402
// max_tenants: 0, // Max number of tenants per ingester. 0 = no limit.
403403
},
404+
405+
// if we disable this, we need to make sure we set the resource limits
406+
// Disabling this can potentially increase cortex performance,
407+
// but it will also cause performance inconsistencies
408+
gomaxprocs_based_on_cpu_requests: true,
409+
gomemlimit_based_on_mem_requests: true,
410+
411+
gomaxprocs_resource:
412+
if $._config.gomaxprocs_based_on_cpu_requests then
413+
'requests.cpu'
414+
else
415+
'limits.cpu',
416+
417+
gomemlimit_resource:
418+
if $._config.gomemlimit_based_on_mem_requests then
419+
'requests.memory'
420+
else
421+
'limits.memory',
404422
},
405423

424+
go_container_mixin::
425+
local container = $.core.v1.container;
426+
container.withEnvMixin([
427+
container.envType.withName('GOMAXPROCS') +
428+
container.envType.valueFrom.resourceFieldRef.withResource($._config.gomaxprocs_resource),
429+
container.envType.withName('GOMEMLIMIT') +
430+
container.envType.valueFrom.resourceFieldRef.withResource($._config.gomemlimit_resource),
431+
]),
432+
406433
local configMap = $.core.v1.configMap,
407434

408435
overrides_config:

‎cortex/distributor.libsonnet‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,11 @@
4444
container.new('distributor', $._images.distributor) +
4545
container.withPorts($.distributor_ports) +
4646
container.withArgsMixin($.util.mapToFlags($.distributor_args)) +
47-
container.withEnvMixin([
48-
envType.withName('GOMAXPROCS') +
49-
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
50-
envType.withName('GOMEMLIMIT') +
51-
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
52-
]) +
5347
container.withEnvMap($.distributor_env_map) +
5448
$.util.resourcesRequests('2', '2Gi') +
5549
$.util.resourcesLimits(null, '4Gi') +
5650
$.util.readinessProbe +
51+
$.go_container_mixin +
5752
$.jaeger_mixin,
5853

5954
local deployment = $.apps.v1.deployment,

‎cortex/flusher-job-blocks.libsonnet‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,10 @@
2525
$.util.resourcesRequests('4', '15Gi') +
2626
$.util.resourcesLimits(null, '25Gi') +
2727
$.util.readinessProbe +
28+
$.go_container_mixin +
2829
$.jaeger_mixin,
2930

3031
flusher_env_map:: {
31-
GOMAXPROCS: '4',
32-
GOMEMLIMIT: '15GiB',
3332
},
3433

3534
flusher_job_func(jobName, pvcName)::

‎cortex/ingester.libsonnet‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,15 +56,10 @@
5656
container.withPorts($.ingester_ports) +
5757
container.withArgsMixin($.util.mapToFlags($.ingester_args)) +
5858
container.withEnvMap($.ingester_env_map) +
59-
container.withEnvMixin([
60-
envType.withName('GOMAXPROCS') +
61-
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
62-
envType.withName('GOMEMLIMIT') +
63-
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
64-
]) +
6559
$.util.resourcesRequests('4', '15Gi') +
6660
$.util.resourcesLimits(null, '25Gi') +
6761
$.util.readinessProbe +
62+
$.go_container_mixin +
6863
$.jaeger_mixin,
6964

7065
ingester_deployment_labels:: {},

‎cortex/overrides-exporter.libsonnet‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
container.withArgsMixin($.util.mapToFlags($.overrides_exporter_args, prefix='--')) +
2121
$.util.resourcesRequests('0.5', '0.5Gi') +
2222
$.util.readinessProbe +
23+
$.go_container_mixin +
2324
container.mixin.readinessProbe.httpGet.withPort($.overrides_exporter_port.name),
2425

2526
local deployment = $.apps.v1.deployment,

‎cortex/querier.libsonnet‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,7 @@
4343
$.jaeger_mixin +
4444
$.util.readinessProbe +
4545
container.withEnvMap($.querier_env_map) +
46-
container.withEnvMixin([
47-
envType.withName('GOMAXPROCS') +
48-
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
49-
envType.withName('GOMEMLIMIT') +
50-
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
51-
]) +
46+
$.go_container_mixin +
5247
$.util.resourcesRequests('2', '12Gi') +
5348
$.util.resourcesLimits(null, '24Gi'),
5449

‎cortex/query-frontend.libsonnet‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,7 @@
4444
container.withPorts($.util.defaultPorts) +
4545
container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) +
4646
container.withEnvMap($.query_frontend_env_map) +
47-
container.withEnvMixin([
48-
envType.withName('GOMAXPROCS') +
49-
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
50-
envType.withName('GOMEMLIMIT') +
51-
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
52-
]) +
47+
$.go_container_mixin +
5348
$.jaeger_mixin +
5449
$.util.readinessProbe +
5550
$.util.resourcesRequests('2', '600Mi') +

0 commit comments

Comments
 (0)