From d4bc0e0fcf2077aff32b50cdd434406f941eb219 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 13 Sep 2023 22:13:58 -0500 Subject: [PATCH] feat(lambda): Lambda calls default timeout after 50 seconds causing longer running lambdas to fail. This adds configurable timeouts for lambda invocations (#6041) (cherry picked from commit 50d9f54fefd7c65c6a877ac54822b17c2cce4488) --- .../ops/InvokeLambdaAtomicOperation.java | 8 ++- .../deploy/ops/LambdaOperationsConfig.java | 29 ++++++++++ .../ops/InvokeLambdaAtomicOperationTest.java | 55 ++++++++++++++----- 3 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/LambdaOperationsConfig.java diff --git a/clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperation.java b/clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperation.java index fa4a3d2725a..1620be6e2e5 100644 --- a/clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperation.java +++ b/clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperation.java @@ -34,10 +34,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import lombok.extern.log4j.Log4j2; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; +@Log4j2 public class InvokeLambdaAtomicOperation extends AbstractLambdaAtomicOperation< InvokeLambdaFunctionDescription, InvokeLambdaFunctionOutputDescription> @@ -45,6 +47,8 @@ public class InvokeLambdaAtomicOperation @Autowired private ArtifactDownloader artifactDownloader; + @Autowired LambdaOperationsConfig operationsConfig; + public InvokeLambdaAtomicOperation(InvokeLambdaFunctionDescription description) { super(description, "INVOKE_LAMBDA_FUNCTION"); } @@ -71,12 +75,14 @@ private InvokeLambdaFunctionOutputDescription invokeFunction( new InvokeRequest() .withFunctionName(functionName) .withLogType(LogType.Tail) - .withPayload(payload); + .withPayload(payload) + .withSdkRequestTimeout(operationsConfig.getInvokeTimeoutMs()); String qualifierRegex = "|[a-zA-Z0-9$_-]+"; if (description.getQualifier().matches(qualifierRegex)) { req.setQualifier(description.getQualifier()); } + log.info("Invoking Lmabda function " + functionName + " and waiting for it to complete"); InvokeResult result = client.invoke(req); String ans = byteBuffer2String(result.getPayload(), Charset.forName("UTF-8")); diff --git a/clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/LambdaOperationsConfig.java b/clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/LambdaOperationsConfig.java new file mode 100644 index 00000000000..6eaf03f6de4 --- /dev/null +++ b/clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/LambdaOperationsConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 Armory, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.clouddriver.lambda.deploy.ops; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties(prefix = "aws.lambda.ops") +@Data +public class LambdaOperationsConfig { + // Matches AWS SDK default value + private int invokeTimeoutMs = 50000; +} diff --git a/clouddriver-lambda/src/test/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperationTest.java b/clouddriver-lambda/src/test/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperationTest.java index 365644184ec..a2992d5f983 100644 --- a/clouddriver-lambda/src/test/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperationTest.java +++ b/clouddriver-lambda/src/test/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperationTest.java @@ -16,6 +16,7 @@ package com.netflix.spinnaker.clouddriver.lambda.deploy.ops; +import static junit.framework.TestCase.assertEquals; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.*; @@ -26,35 +27,61 @@ import com.netflix.spinnaker.clouddriver.lambda.deploy.description.InvokeLambdaFunctionDescription; import com.netflix.spinnaker.clouddriver.lambda.deploy.description.InvokeLambdaFunctionOutputDescription; import com.netflix.spinnaker.clouddriver.lambda.provider.view.LambdaFunctionProvider; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.springframework.test.util.ReflectionTestUtils; public class InvokeLambdaAtomicOperationTest implements LambdaTestingDefaults { - @Test - void testInvokeLambda() { + InvokeLambdaAtomicOperation invokeOperation; + LambdaOperationsConfig operationsConfig; + + @BeforeEach + public void setup() { InvokeLambdaFunctionDescription invokeDesc = new InvokeLambdaFunctionDescription(); invokeDesc.setFunctionName(fName).setQualifier(version).setRegion(region).setAccount(account); - - InvokeLambdaAtomicOperation invokeOperation = spy(new InvokeLambdaAtomicOperation(invokeDesc)); - doNothing().when(invokeOperation).updateTaskStatus(anyString()); - - AWSLambda lambdaClient = mock(AWSLambda.class); - LambdaFunction cachedFunction = getMockedFunctionDefintion(); + invokeDesc.setPayload("example"); + invokeOperation = spy(new InvokeLambdaAtomicOperation(invokeDesc)); LambdaFunctionProvider lambdaFunctionProvider = mock(LambdaFunctionProvider.class); ReflectionTestUtils.setField(invokeOperation, "lambdaFunctionProvider", lambdaFunctionProvider); - - doReturn(lambdaClient).when(invokeOperation).getLambdaClient(); + LambdaFunction cachedFunction = getMockedFunctionDefintion(); doReturn(cachedFunction) .when(lambdaFunctionProvider) .getFunction(anyString(), anyString(), anyString()); + operationsConfig = new LambdaOperationsConfig(); + ReflectionTestUtils.setField(invokeOperation, "operationsConfig", operationsConfig); + doNothing().when(invokeOperation).updateTaskStatus(anyString()); + } - InvokeRequest invokeRequest = new InvokeRequest(); - invokeRequest.withQualifier(version).withFunctionName(functionArn); - InvokeResult mockDeleteResult = new InvokeResult(); - doReturn(mockDeleteResult).when(lambdaClient).invoke(invokeRequest); + @Test + void testInvokeLambda() { + + AWSLambda lambdaClient = mock(AWSLambda.class); + doReturn(lambdaClient).when(invokeOperation).getLambdaClient(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(InvokeRequest.class); + InvokeResult result = new InvokeResult(); + doReturn(result).when(lambdaClient).invoke(any(InvokeRequest.class)); InvokeLambdaFunctionOutputDescription output = invokeOperation.operate(null); + assertEquals(result, output.getInvokeResult()); + verify(lambdaClient).invoke(captor.capture()); verify(invokeOperation, atLeastOnce()).updateTaskStatus(anyString()); + assertEquals(fName, captor.getValue().getFunctionName()); + assertEquals(50000, captor.getValue().getSdkRequestTimeout().intValue()); + } + + @Test + void verifyTimeoutIsSet() { + operationsConfig.setInvokeTimeoutMs(100000); + + AWSLambda lambdaClient = mock(AWSLambda.class); + doReturn(lambdaClient).when(invokeOperation).getLambdaClient(); + + ArgumentCaptor captor = ArgumentCaptor.forClass(InvokeRequest.class); + doReturn(new InvokeResult()).when(lambdaClient).invoke(captor.capture()); + invokeOperation.operate(null); + assertEquals(100000, captor.getValue().getSdkRequestTimeout().intValue()); } }