feat(lambda): Lambda calls default timeout after 50 seconds causing l…

…onger running lambdas to fail. This adds configurable timeouts for lambda invocations (#6041)

(cherry picked from commit 50d9f54)

clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperation.java

@@ -34,17 +34,21 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
+import lombok.extern.log4j.Log4j2;
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.commons.io.FileUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 
+@Log4j2
 public class InvokeLambdaAtomicOperation
     extends AbstractLambdaAtomicOperation<
         InvokeLambdaFunctionDescription, InvokeLambdaFunctionOutputDescription>
     implements AtomicOperation<InvokeLambdaFunctionOutputDescription> {
 
   @Autowired private ArtifactDownloader artifactDownloader;
 
+  @Autowired LambdaOperationsConfig operationsConfig;
+
   public InvokeLambdaAtomicOperation(InvokeLambdaFunctionDescription description) {
     super(description, "INVOKE_LAMBDA_FUNCTION");
   }
@@ -71,12 +75,14 @@ private InvokeLambdaFunctionOutputDescription invokeFunction(
         new InvokeRequest()
             .withFunctionName(functionName)
             .withLogType(LogType.Tail)
-            .withPayload(payload);
+            .withPayload(payload)
+            .withSdkRequestTimeout(operationsConfig.getInvokeTimeoutMs());
 
     String qualifierRegex = "|[a-zA-Z0-9$_-]+";
     if (description.getQualifier().matches(qualifierRegex)) {
       req.setQualifier(description.getQualifier());
     }
+    log.info("Invoking Lmabda function " + functionName + " and waiting for it to complete");
 
     InvokeResult result = client.invoke(req);
     String ans = byteBuffer2String(result.getPayload(), Charset.forName("UTF-8"));

clouddriver-lambda/src/main/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/LambdaOperationsConfig.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2021 Armory, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.lambda.deploy.ops;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConfigurationProperties(prefix = "aws.lambda.ops")
+@Data
+public class LambdaOperationsConfig {
+  // Matches AWS SDK default value
+  private int invokeTimeoutMs = 50000;
+}

clouddriver-lambda/src/test/java/com/netflix/spinnaker/clouddriver/lambda/deploy/ops/InvokeLambdaAtomicOperationTest.java

@@ -16,6 +16,7 @@
 
 package com.netflix.spinnaker.clouddriver.lambda.deploy.ops;
 
+import static junit.framework.TestCase.assertEquals;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.*;
 
@@ -26,35 +27,61 @@
 import com.netflix.spinnaker.clouddriver.lambda.deploy.description.InvokeLambdaFunctionDescription;
 import com.netflix.spinnaker.clouddriver.lambda.deploy.description.InvokeLambdaFunctionOutputDescription;
 import com.netflix.spinnaker.clouddriver.lambda.provider.view.LambdaFunctionProvider;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 import org.springframework.test.util.ReflectionTestUtils;
 
 public class InvokeLambdaAtomicOperationTest implements LambdaTestingDefaults {
-  @Test
-  void testInvokeLambda() {
+  InvokeLambdaAtomicOperation invokeOperation;
+  LambdaOperationsConfig operationsConfig;
+
+  @BeforeEach
+  public void setup() {
     InvokeLambdaFunctionDescription invokeDesc = new InvokeLambdaFunctionDescription();
     invokeDesc.setFunctionName(fName).setQualifier(version).setRegion(region).setAccount(account);
-
-    InvokeLambdaAtomicOperation invokeOperation = spy(new InvokeLambdaAtomicOperation(invokeDesc));
-    doNothing().when(invokeOperation).updateTaskStatus(anyString());
-
-    AWSLambda lambdaClient = mock(AWSLambda.class);
-    LambdaFunction cachedFunction = getMockedFunctionDefintion();
+    invokeDesc.setPayload("example");
+    invokeOperation = spy(new InvokeLambdaAtomicOperation(invokeDesc));
 
     LambdaFunctionProvider lambdaFunctionProvider = mock(LambdaFunctionProvider.class);
     ReflectionTestUtils.setField(invokeOperation, "lambdaFunctionProvider", lambdaFunctionProvider);
-
-    doReturn(lambdaClient).when(invokeOperation).getLambdaClient();
+    LambdaFunction cachedFunction = getMockedFunctionDefintion();
     doReturn(cachedFunction)
         .when(lambdaFunctionProvider)
         .getFunction(anyString(), anyString(), anyString());
+    operationsConfig = new LambdaOperationsConfig();
+    ReflectionTestUtils.setField(invokeOperation, "operationsConfig", operationsConfig);
+    doNothing().when(invokeOperation).updateTaskStatus(anyString());
+  }
 
-    InvokeRequest invokeRequest = new InvokeRequest();
-    invokeRequest.withQualifier(version).withFunctionName(functionArn);
-    InvokeResult mockDeleteResult = new InvokeResult();
-    doReturn(mockDeleteResult).when(lambdaClient).invoke(invokeRequest);
+  @Test
+  void testInvokeLambda() {
+
+    AWSLambda lambdaClient = mock(AWSLambda.class);
+    doReturn(lambdaClient).when(invokeOperation).getLambdaClient();
+
+    ArgumentCaptor<InvokeRequest> captor = ArgumentCaptor.forClass(InvokeRequest.class);
+    InvokeResult result = new InvokeResult();
+    doReturn(result).when(lambdaClient).invoke(any(InvokeRequest.class));
 
     InvokeLambdaFunctionOutputDescription output = invokeOperation.operate(null);
+    assertEquals(result, output.getInvokeResult());
+    verify(lambdaClient).invoke(captor.capture());
     verify(invokeOperation, atLeastOnce()).updateTaskStatus(anyString());
+    assertEquals(fName, captor.getValue().getFunctionName());
+    assertEquals(50000, captor.getValue().getSdkRequestTimeout().intValue());
+  }
+
+  @Test
+  void verifyTimeoutIsSet() {
+    operationsConfig.setInvokeTimeoutMs(100000);
+
+    AWSLambda lambdaClient = mock(AWSLambda.class);
+    doReturn(lambdaClient).when(invokeOperation).getLambdaClient();
+
+    ArgumentCaptor<InvokeRequest> captor = ArgumentCaptor.forClass(InvokeRequest.class);
+    doReturn(new InvokeResult()).when(lambdaClient).invoke(captor.capture());
+    invokeOperation.operate(null);
+    assertEquals(100000, captor.getValue().getSdkRequestTimeout().intValue());
   }
 }