GH-923: Fix start delay for @lazy @RabbitListener

Fixes #923

While checking for missing or mis-matched queues, a lazily-loaded
listener container can deadlock for 60 seconds.

This occurs if the `allBeanNamesByType` cache does not currently have
an entry for `Queue` (e.g. cleared by registering a singleton).
When lazy beans are referenced, the `RabbitListenerEndpointRegistry`
starts the container and `start()` waits for the consumers to start.
Getting a reference to the lazy bean holds the `singletonObjects`
lock, which is required by the consumer(s) to get the `Queue` beans
to check.

Add a test case to demonstrate the issue.

Disable the redeclaration logic during the initial start of such a
container.

**cherry-pick to 2.1.x**

Author: garyrussell

spring-rabbit/src/main/java/org/springframework/amqp/rabbit/listener/AbstractMessageListenerContainer.java

@@ -214,6 +214,8 @@ public abstract class AbstractMessageListenerContainer extends RabbitAccessor
 
 	private String errorHandlerLoggerName = getClass().getName();
 
+	private volatile boolean lazyLoad;
+
 	/**
 	 * {@inheritDoc}
 	 * @since 1.5
@@ -1190,6 +1192,9 @@ public void start() {
 		catch (Exception ex) {
 			throw convertRabbitAccessException(ex);
 		}
+		finally {
+			this.lazyLoad = false;
+		}
 	}
 
 	/**
@@ -1602,6 +1607,25 @@ protected void checkMismatchedQueues() {
 		}
 	}
 
+	@Override
+	public void lazyLoad() {
+		if (this.mismatchedQueuesFatal) {
+			if (this.missingQueuesFatal) {
+				logger.warn("'mismatchedQueuesFatal' and 'missingQueuesFatal' are ignored during the initial start(), "
+						+ "for lazily loaded containers");
+			}
+			else {
+				logger.warn("'mismatchedQueuesFatal' is ignored during the initial start(), "
+						+ "for lazily loaded containers");
+			}
+		}
+		else if (this.missingQueuesFatal) {
+			logger.warn("'missingQueuesFatal' is ignored during the initial start(), "
+					+ "for lazily loaded containers");
+		}
+		this.lazyLoad = true;
+	}
+
 	/**
 	 * Use {@link RabbitAdmin#initialize()} to redeclare everything if necessary.
 	 * Since auto deletion of a queue can cause upstream elements
@@ -1621,7 +1645,7 @@ protected void checkMismatchedQueues() {
 	 */
 	protected synchronized void redeclareElementsIfNecessary() {
 		RabbitAdmin rabbitAdmin = getRabbitAdmin();
-		if (rabbitAdmin == null || !isAutoDeclare()) {
+		if (this.lazyLoad || rabbitAdmin == null || !isAutoDeclare()) {
 			return;
 		}
 		try {

spring-rabbit/src/main/java/org/springframework/amqp/rabbit/listener/MessageListenerContainer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 the original author or authors.
+ * Copyright 2014-2019 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -48,4 +48,15 @@ public interface MessageListenerContainer extends SmartLifecycle {
 	@Deprecated
 	MessageConverter getMessageConverter();
 
+	/**
+	 * Do not check for missing or mismatched queues during startup. Used for lazily
+	 * loaded message listener containers to avoid a deadlock when starting such
+	 * containers. Applications lazily loading containers should verify the queue
+	 * configuration before loading the container bean.
+	 * @since 2.1.5
+	 */
+	default void lazyLoad() {
+		// no-op
+	}
+
 }

spring-rabbit/src/main/java/org/springframework/amqp/rabbit/listener/RabbitListenerEndpointRegistry.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2018 the original author or authors.
+ * Copyright 2014-2019 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -164,6 +164,9 @@ public void registerListenerContainer(RabbitListenerEndpoint endpoint, RabbitLis
 				}
 				containerGroup.add(container);
 			}
+			if (this.contextRefreshed) {
+				container.lazyLoad();
+			}
 			if (startImmediately) {
 				startIfNecessary(container);
 			}

spring-rabbit/src/test/java/org/springframework/amqp/rabbit/annotation/LazyContainerTests.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2019 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.amqp.rabbit.annotation;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.lessThan;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.springframework.amqp.core.Queue;
+import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory;
+import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
+import org.springframework.amqp.rabbit.core.RabbitAdmin;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.junit.BrokerRunning;
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringRunner;
+
+/**
+ * @author Gary Russell
+ * @since 2.1.5
+ *
+ */
+@RunWith(SpringRunner.class)
+@DirtiesContext
+public class LazyContainerTests {
+
+	@ClassRule
+	public static BrokerRunning brokerRunning = BrokerRunning.isRunningWithEmptyQueues("test.lazy");
+
+	@Autowired
+	private ConfigurableApplicationContext context;
+
+	@Autowired
+	private ObjectProvider<LazyListener> lazyListenerProvider;
+
+	@Autowired
+	private  RabbitTemplate rabbitTemplate;
+
+	@Test
+	public void lazy() {
+		this.context.getBeanFactory().registerSingleton("clearTheByTypeCache", "foo");
+		long t1 = System.currentTimeMillis();
+		this.lazyListenerProvider.getIfAvailable();
+		assertThat(System.currentTimeMillis() - t1, lessThan(30_000L));
+		Object reply = this.rabbitTemplate.convertSendAndReceive("test.lazy", "lazy");
+		assertNotNull(reply);
+		assertThat(reply, equalTo("LAZY"));
+	}
+
+	@Configuration
+	@EnableRabbit
+	public static class Config {
+
+		@Bean
+		public CachingConnectionFactory cf() {
+			return new CachingConnectionFactory(brokerRunning.getConnectionFactory());
+		}
+
+		@Bean
+		public RabbitTemplate template() {
+			return new RabbitTemplate(cf());
+		}
+
+		@Bean
+		public RabbitAdmin admin() {
+			return new RabbitAdmin(template());
+		}
+
+		@Bean
+		public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory() {
+			SimpleRabbitListenerContainerFactory cf = new SimpleRabbitListenerContainerFactory();
+			cf.setConnectionFactory(cf());
+			return cf;
+		}
+
+		@Bean
+		public Queue queue() {
+			return new Queue("test.lazy");
+		}
+
+		@Bean
+		@Lazy
+		public LazyListener listener() {
+			return new LazyListener();
+		}
+
+		@RabbitListener(queues = "test.lazy")
+		public String listen(String in) {
+			return in.toUpperCase();
+		}
+
+	}
+
+	public static class LazyListener {
+
+		@RabbitListener(queues = "test.lazy", concurrency = "2")
+		public String listenLazily(String in) {
+			return in.toUpperCase();
+		}
+
+	}
+
+}

src/reference/asciidoc/amqp.adoc

@@ -4611,6 +4611,10 @@ This global property will not be applied to any containers that have an explicit
 
 The default retry properties (3 retries at 5 second intervals) can be overridden using the properties below.
 
+IMPORTANT: Missing queue detection is disabled while starting a container for a `@RabbitListener` in a bean that is marked `@Lazy`.
+This is to avoid a potential deadlock which can delay the start of such containers for up to 60 seconds.
+Applications using lazy listener beans should check the queue(s) before getting a reference to the lazy bean.
+
 a| image::images/tickmark.png[]
 a| image::images/tickmark.png[]
 
@@ -4669,6 +4673,10 @@ If you wish to limit the checks to just those queues used by a container, you sh
 `RabbitAdmin` for the container, and provide a reference to it using the `rabbitAdmin` property.
 See <<conditional-declaration>> for more information.
 
+IMPORTANT: Mismatched queue argument detection is disabled while starting a container for a `@RabbitListener` in a bean that is marked `@Lazy`.
+This is to avoid a potential deadlock which can delay the start of such containers for up to 60 seconds.
+Applications using lazy listener beans should check the queue arguments before getting a reference to the lazy bean.
+
 a| image::images/tickmark.png[]
 a| image::images/tickmark.png[]