Test fleet, allocator-client, and custom 'players' per game server co…

…unter metric to test allocation for replicating issue #3992

examples/allocator-client/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
 func main() {
@@ -55,6 +56,36 @@ func main() {
 
 	request := &pb.AllocationRequest{
 		Namespace: *namespace,
+		GameServerSelectors: []*pb.GameServerSelector{
+			{
+				GameServerState: pb.GameServerSelector_ALLOCATED,
+				MatchLabels: map[string]string{
+					"version": "1.2.3",
+				},
+				Counters: map[string]*pb.CounterSelector{
+					"players": {
+						MinAvailable: 1,
+					},
+				},
+			},
+			{
+				GameServerState: pb.GameServerSelector_READY,
+				MatchLabels: map[string]string{
+					"version": "1.2.3",
+				},
+				Counters: map[string]*pb.CounterSelector{
+					"players": {
+						MinAvailable: 1,
+					},
+				},
+			},
+		},
+		Counters: map[string]*pb.CounterAction{
+			"players": {
+				Action: wrapperspb.String("Increment"),
+				Amount: wrapperspb.Int64(1),
+			},
+		},
 		MultiClusterSetting: &pb.MultiClusterSetting{
 			Enabled: *multicluster,
 		},

pkg/metrics/controller.go

@@ -62,7 +62,7 @@ const (
 
 var (
 	// MetricResyncPeriod is the interval to re-synchronize metrics based on indexed cache.
-	MetricResyncPeriod = time.Second * 15
+	MetricResyncPeriod = time.Second * 1
 )
 
 func init() {
@@ -449,7 +449,15 @@ func (c *Controller) recordGameServerStatusChanges(old, next interface{}) {
 			RecordWithTags(context.Background(), []tag.Mutator{tag.Upsert(keyFleetName, fleetName),
 				tag.Upsert(keyName, newGs.GetName()), tag.Upsert(keyNamespace, newGs.GetNamespace())}, gameServerPlayerCapacityTotal.M(newGs.Status.Players.Capacity-newGs.Status.Players.Count))
 		}
+	}
 
+	if runtime.FeatureEnabled(runtime.FeatureCountsAndLists) && len(newGs.Status.Counters) != 0 {
+		if counterStatus, ok := newGs.Status.Counters["players"]; ok {
+			if counterStatus.Count != oldGs.Status.Counters["players"].Count {
+				RecordWithTags(context.Background(), []tag.Mutator{tag.Upsert(keyFleetName, fleetName),
+					tag.Upsert(keyName, newGs.GetName()), tag.Upsert(keyNamespace, newGs.GetNamespace())}, gameServersCountersStats.M(counterStatus.Count))
+			}
+		}
 	}
 
 	if newGs.Status.State != oldGs.Status.State {

pkg/metrics/controller_metrics.go

@@ -31,6 +31,7 @@ const (
 	fleetAutoscalersLimitedName             = "fleet_autoscalers_limited"
 	fleetCountersName                       = "fleet_counters"
 	fleetListsName                          = "fleet_lists"
+	gameServersCountersName                 = "gameservers_counters"
 	gameServersCountName                    = "gameservers_count"
 	gameServersTotalName                    = "gameservers_total"
 	gameServersPlayerConnectedTotalName     = "gameserver_player_connected_total"
@@ -45,7 +46,7 @@ var (
 	fleetAutoscalerViews = []string{fleetAutoscalerBufferLimitName, fleetAutoscalterBufferSizeName, fleetAutoscalerCurrentReplicaCountName,
 		fleetAutoscalersDesiredReplicaCountName, fleetAutoscalersAbleToScaleName, fleetAutoscalersLimitedName}
 	// fleetViews are metric views associated with Fleets
-	fleetViews = append([]string{fleetRolloutPercent, fleetReplicaCountName, gameServersCountName, gameServersTotalName, gameServersPlayerConnectedTotalName, gameServersPlayerCapacityTotalName, gameServerStateDurationName, fleetCountersName, fleetListsName}, fleetAutoscalerViews...)
+	fleetViews = append([]string{fleetRolloutPercent, fleetReplicaCountName, gameServersCountersName, gameServersCountName, gameServersTotalName, gameServersPlayerConnectedTotalName, gameServersPlayerCapacityTotalName, gameServerStateDurationName, fleetCountersName, fleetListsName}, fleetAutoscalerViews...)
 
 	stateDurationSeconds           = []float64{0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384}
 	fleetRolloutPercentStats       = stats.Int64("fleets/rollout_percent", "The current fleet rollout percentage", "1")
@@ -58,6 +59,7 @@ var (
 	fasLimitedStats                = stats.Int64("fas/limited", "The fleet autoscaler is capped (0 indicates false, 1 indicates true)", "1")
 	fleetCountersStats             = stats.Int64("fleets/counters", "Aggregated Counters counts and capacity across GameServers in the Fleet", "1")
 	fleetListsStats                = stats.Int64("fleets/lists", "Aggregated Lists counts and capacity across GameServers in the Fleet", "1")
+	gameServersCountersStats       = stats.Int64("gameservers/counters", "Counters connected to gameservers", "1")
 	gameServerCountStats           = stats.Int64("gameservers/count", "The count of gameservers", "1")
 	gameServerTotalStats           = stats.Int64("gameservers/total", "The total of gameservers", "1")
 	gameServerPlayerConnectedTotal = stats.Int64("gameservers/player_connected", "The total number of players connected to gameservers", "1")
@@ -137,6 +139,13 @@ var (
 			Aggregation: view.LastValue(),
 			TagKeys:     []tag.Key{keyName, keyNamespace, keyType, keyList},
 		},
+		{
+			Name:        gameServersCountersName,
+			Measure:     gameServersCountersStats,
+			Description: "The current count of counters in gameservers",
+			Aggregation: view.LastValue(),
+			TagKeys:     []tag.Key{keyFleetName, keyName, keyNamespace},
+		},
 		{
 			Name:        gameServersCountName,
 			Measure:     gameServerCountStats,

pkg/metrics/controller_test.go

@@ -158,6 +158,58 @@ func TestControllerGameServerCount(t *testing.T) {
 	})
 }
 
+func TestControllerGameServerCountersCount(t *testing.T) {
+	runtime.FeatureTestMutex.Lock()
+	defer runtime.FeatureTestMutex.Unlock()
+	runtime.EnableAllFeatures()
+	resetMetrics()
+	exporter := &metricExporter{}
+	reader := metricexport.NewReader()
+
+	c := newFakeController()
+	defer c.close()
+
+	gs1 := gameServerWithFleetAndState("test-fleet", agonesv1.GameServerStateReady)
+	gs1.Status.Counters["players"] = agonesv1.CounterStatus{Count: 0, Capacity: 10}
+	c.gsWatch.Add(gs1)
+	gs1 = gs1.DeepCopy()
+	playerCounter := gs1.Status.Counters["players"]
+	playerCounter.Count++
+	gs1.Status.Counters["players"] = playerCounter
+	c.gsWatch.Modify(gs1)
+
+	c.run(t)
+	require.True(t, c.sync())
+	require.Eventually(t, func() bool {
+		gs, err := c.gameServerLister.GameServers(gs1.ObjectMeta.Namespace).Get(gs1.ObjectMeta.Name)
+		assert.NoError(t, err)
+		pc := gs.Status.Counters["players"]
+		return pc.Count == 1
+	}, 5*time.Second, time.Second)
+	c.collect()
+
+	gs1 = gs1.DeepCopy()
+	playerCounter = gs1.Status.Counters["players"]
+	playerCounter.Count += 4
+	gs1.Status.Counters["players"] = playerCounter
+	c.gsWatch.Modify(gs1)
+
+	c.run(t)
+	require.True(t, c.sync())
+	require.Eventually(t, func() bool {
+		gs, err := c.gameServerLister.GameServers(gs1.ObjectMeta.Namespace).Get(gs1.ObjectMeta.Name)
+		assert.NoError(t, err)
+		pc := gs.Status.Counters["players"]
+		return pc.Count == 5
+	}, 5*time.Second, time.Second)
+	c.collect()
+
+	reader.ReadAndExport(exporter)
+	assertMetricData(t, exporter, gameServersCountersName, []expectedMetricData{
+		{labels: []string{"test-fleet", gs1.GetName(), defaultNs}, val: int64(5)},
+	})
+}
+
 func TestControllerGameServerPlayerConnectedCount(t *testing.T) {
 	runtime.FeatureTestMutex.Lock()
 	defer runtime.FeatureTestMutex.Unlock()

pkg/metrics/util_test.go

@@ -140,7 +140,8 @@ func gameServerWithFleetAndState(fleetName string, state agonesv1.GameServerStat
 			Labels:    lbs,
 		},
 		Status: agonesv1.GameServerStatus{
-			State: state,
+			State:    state,
+			Counters: map[string]agonesv1.CounterStatus{},
 		},
 	}
 	return gs

tmp/allocate.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Get the key, cert, and tls files from "Send allocation request" instructions
+# https://agones.dev/site/docs/advanced/allocator-service/
+NAMESPACE=default
+EXTERNAL_IP=$(kubectl get services agones-allocator -n agones-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+KEY_FILE=client.key
+CERT_FILE=client.crt
+TLS_CA_FILE=ca.crt
+
+echo "Starting go runs"
+
+for _ in {1..10000}; do
+  go run ../examples/allocator-client/main.go \
+    --ip "${EXTERNAL_IP}" \
+    --port 443 \
+    --namespace "${NAMESPACE}" \
+    --key "${KEY_FILE}" \
+    --cert "${CERT_FILE}" \
+    --cacert "${TLS_CA_FILE}"
+done
+
+echo "All done"

tmp/fleet.yaml

@@ -0,0 +1,43 @@
+---
+# Copyright 2020 Google LLC All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: agones.dev/v1
+kind: Fleet
+metadata:
+  name: simple-game-server
+spec:
+  replicas: 300
+  template:
+    metadata:
+      labels:
+        version: "1.2.3"
+    spec:
+      ports:
+        - name: default
+          containerPort: 7654
+      counters:
+        players:
+          capacity: 300
+      template:
+        spec:
+          containers:
+            - name: simple-game-server
+              image: us-docker.pkg.dev/agones-images/examples/simple-game-server:0.35
+              resources:
+                requests:
+                  memory: 32Mi
+                  cpu: 10m
+                limits:
+                  memory: 32Mi
+                  cpu: 10m