diff --git a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/configdata/NacosConfigDataLoader.java b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/configdata/NacosConfigDataLoader.java index 932640314..c4628c5cc 100644 --- a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/configdata/NacosConfigDataLoader.java +++ b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/configdata/NacosConfigDataLoader.java @@ -104,11 +104,11 @@ public ConfigData doLoad(ConfigDataLoaderContext context, private ConfigService getConfigService(ConfigDataLoaderContext context) { NacosConfigServiceMasterManager configServiceMaster = getBean(context, NacosConfigServiceMasterManager.class); - if (configServiceMaster != null && configServiceMaster.checkServerConnect()) { + if (configServiceMaster != null && configServiceMaster.isNacosServerHealth()) { return configServiceMaster.getConfigService(); } NacosConfigServiceStandbyManager configServiceStandby = getBean(context, NacosConfigServiceStandbyManager.class); - if (configServiceStandby != null && configServiceStandby.checkServerConnect()) { + if (configServiceStandby != null && configServiceStandby.isNacosServerHealth()) { return configServiceStandby.getConfigService(); } if (configServiceMaster != null) { diff --git a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/refresh/NacosContextRefresher.java b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/refresh/NacosContextRefresher.java index d58c5df74..e770bd368 100644 --- a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/refresh/NacosContextRefresher.java +++ b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/alibaba/cloud/nacos/refresh/NacosContextRefresher.java @@ -143,7 +143,8 @@ private void startSchedulerTask() { private void masterStandbyServerCheck() { synchronized (NacosContextRefresher.class) { - if (!NacosConfigConst.STATUS_UP.equals(currentConfigServiceManager.getConfigService().getServerStatus())) { + // both check interface invoking and GRPC connect for fail count. + if (!currentConfigServiceManager.isNacosServerHealth() || !currentConfigServiceManager.isRpcConnectHealth()) { if (failCount == 2) { reRegisterNacosListeners(); failCount = 0; @@ -157,13 +158,22 @@ private void masterStandbyServerCheck() { // if current server is not master, check whether the master status is healthy, if status UP, use master server. if (isRetryMasterServer - && !currentConfigServiceManager.getServerAddr().equals(nacosConfigManagers.get(0).getServerAddr()) - && nacosConfigManagers.get(0).checkServerConnect()) { + && !currentConfigServiceManager.isMasterConfigService() + && getMasterConfigService().isNacosServerHealth()) { reRegisterNacosListeners(); } } } + private NacosConfigManager getMasterConfigService() { + for (NacosConfigManager configManager : nacosConfigManagers) { + if (configManager.isMasterConfigService()) { + return configManager; + } + } + return nacosConfigManagers.get(0); + } + private void reRegisterNacosListeners() { shutDownConfigService(currentConfigServiceManager); clearConfigListener(); diff --git a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/client/NacosPropertiesFuzzyQueryService.java b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/client/NacosPropertiesFuzzyQueryService.java index a26d2ff71..593310251 100644 --- a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/client/NacosPropertiesFuzzyQueryService.java +++ b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/client/NacosPropertiesFuzzyQueryService.java @@ -17,26 +17,23 @@ package com.huaweicloud.nacos.config.client; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; import com.alibaba.cloud.nacos.NacosConfigProperties; -import com.alibaba.nacos.client.auth.impl.NacosAuthLoginConstant; -import com.alibaba.nacos.client.auth.impl.process.HttpLoginProcessor; -import com.alibaba.nacos.client.config.impl.ConfigHttpClientManager; import com.alibaba.nacos.common.http.HttpRestResult; import com.alibaba.nacos.common.http.client.NacosRestTemplate; import com.alibaba.nacos.common.http.param.Header; import com.alibaba.nacos.common.http.param.Query; -import com.alibaba.nacos.plugin.auth.api.LoginIdentityContext; import com.huaweicloud.nacos.config.NacosConfigConst; import com.huaweicloud.nacos.config.manager.ConfigServiceManagerUtils; @@ -45,19 +42,11 @@ public class NacosPropertiesFuzzyQueryService { private static final String NACOS_CONFIG_QUERY_URI = "%s/nacos/v1/cs/configs"; - private NacosConfigProperties configProperties; - - private Properties properties; - - private final Map address_token = new ConcurrentHashMap<>(); - - private long tokenTtl; - - private long lastRefreshTime; + private static final NacosPropertiesFuzzyQueryService INSTANCE = new NacosPropertiesFuzzyQueryService(); - private long refreshWindow; + private NacosConfigProperties configProperties; - private static final NacosPropertiesFuzzyQueryService INSTANCE = new NacosPropertiesFuzzyQueryService(); + private final Map addressPropertiesMap = new HashMap<>(); private NacosPropertiesFuzzyQueryService() { @@ -68,16 +57,24 @@ public static NacosPropertiesFuzzyQueryService getInstance() { } public List loadRouterProperties() { - try { - String address = chooseAddress(); - NacosRestTemplate nacosRestTemplate = ConfigHttpClientManager.getInstance().getNacosRestTemplate(); - HttpRestResult response = nacosRestTemplate.get(buildUrl(address), initHeader(address), - buildQuery(), PropertiePageQueryResult.class); - return response.getData().getPageItems(); - } catch (Exception e) { - LOGGER.error("load router properties failed!", e); - return Collections.emptyList(); + buildAddressPropertiesMap(); + List addresses = getAddresses(); + for (String address : addresses) { + try { + NacosRestTemplate nacosRestTemplate = ConfigServiceManagerUtils.getNacosRestTemplate(); + String url = ConfigServiceManagerUtils.buildUrl(address, NACOS_CONFIG_QUERY_URI); + Header header = ConfigServiceManagerUtils.initHeader(address, configProperties.getUsername(), + addressPropertiesMap.get(address)); + HttpRestResult response = nacosRestTemplate.get(url, header, buildQuery(), + PropertiePageQueryResult.class); + if (response.getData() != null && !CollectionUtils.isEmpty(response.getData().getPageItems())) { + return response.getData().getPageItems(); + } + } catch (Exception e) { + LOGGER.error("load router properties failed!", e); + } } + return Collections.emptyList(); } private Query buildQuery() { @@ -93,60 +90,26 @@ private Query buildQuery() { return query; } - private Header initHeader(String address) { - Header header = Header.newInstance(); - if (!StringUtils.isEmpty(configProperties.getUsername())) { - header.addParam(NacosAuthLoginConstant.ACCESSTOKEN, getAccessToken(address)); - } - return header; - } - - private String buildUrl(String address) { - String prefix = ""; - if (!configProperties.getServerAddr().startsWith("http")) { - prefix = "http://"; - } - return prefix + String.format(NACOS_CONFIG_QUERY_URI, address); - } - private String getNamespace() { return StringUtils.isEmpty(configProperties.getNamespace()) ? "" : configProperties.getNamespace(); } - private String chooseAddress() { - properties = configProperties.assembleMasterNacosServerProperties(); - if (!configProperties.isMasterStandbyEnabled()) { - return configProperties.getServerAddr(); - } - if (ConfigServiceManagerUtils.checkServerConnect(configProperties.getServerAddr())) { - return configProperties.getServerAddr(); + private List getAddresses() { + List addresses = new ArrayList<>(); + addresses.add(configProperties.getServerAddr()); + if (configProperties.isMasterStandbyEnabled() && !StringUtils.isEmpty(configProperties.getStandbyServerAddr())) { + addresses.add(configProperties.getStandbyServerAddr()); } - if (ConfigServiceManagerUtils.checkServerConnect(configProperties.getStandbyServerAddr())) { - properties = configProperties.assembleStandbyNacosServerProperties(); - return configProperties.getStandbyServerAddr(); - } - return configProperties.getServerAddr(); + return addresses; } - private String getAccessToken(String address) { - if (address_token.get(address) != null - && (System.currentTimeMillis() - lastRefreshTime) < TimeUnit.SECONDS.toMillis(tokenTtl - refreshWindow)) { - return address_token.get(address); - } - HttpLoginProcessor httpLoginProcessor - = new HttpLoginProcessor(ConfigHttpClientManager.getInstance().getNacosRestTemplate()); - properties.setProperty(NacosAuthLoginConstant.SERVER, address); - LoginIdentityContext identityContext = httpLoginProcessor.getResponse(properties); - if (identityContext != null - && !StringUtils.isEmpty(identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN))) { - tokenTtl = Long.parseLong(identityContext.getParameter(NacosAuthLoginConstant.TOKENTTL)); - refreshWindow = tokenTtl / 10; - lastRefreshTime = System.currentTimeMillis(); - address_token.put(address, identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN)); - return identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN); + private void buildAddressPropertiesMap() { + Properties masterProperties = configProperties.assembleMasterNacosServerProperties(); + addressPropertiesMap.put(configProperties.getServerAddr(), masterProperties); + if (!StringUtils.isEmpty(configProperties.getStandbyServerAddr())) { + Properties standbyProperties = configProperties.assembleStandbyNacosServerProperties(); + addressPropertiesMap.put(configProperties.getStandbyServerAddr(), standbyProperties); } - lastRefreshTime = System.currentTimeMillis(); - return ""; } public void setConfigProperties(NacosConfigProperties nacosConfigProperties) { diff --git a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/ConfigServiceManagerUtils.java b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/ConfigServiceManagerUtils.java index 3257d75ec..973f25ac3 100644 --- a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/ConfigServiceManagerUtils.java +++ b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/ConfigServiceManagerUtils.java @@ -17,20 +17,48 @@ package com.huaweicloud.nacos.config.manager; +import static com.alibaba.nacos.api.PropertyKeyConst.USERNAME; + import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; import java.net.URI; import java.net.URISyntaxException; import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.alibaba.nacos.client.auth.impl.NacosAuthLoginConstant; +import com.alibaba.nacos.client.auth.impl.process.HttpLoginProcessor; +import com.alibaba.nacos.client.config.impl.ConfigHttpClientManager; +import com.alibaba.nacos.common.http.HttpRestResult; +import com.alibaba.nacos.common.http.client.NacosRestTemplate; +import com.alibaba.nacos.common.http.param.Header; +import com.alibaba.nacos.common.http.param.Query; +import com.alibaba.nacos.plugin.auth.api.LoginIdentityContext; + public class ConfigServiceManagerUtils { private static final Logger LOGGER = LoggerFactory.getLogger(ConfigServiceManagerUtils.class); + private static final Map address_token = new ConcurrentHashMap<>(); + + private static final String NACOS_CONFIG_HEALTH_CHECK_URI = "%s/nacos/v1/console/health/config/readiness"; + + private static long tokenTtl; + + private static long lastRefreshTime; + + private static long refreshWindow; + + private static volatile NacosRestTemplate nacosRestTemplate; + public static boolean checkServerConnect(String serverAddress) { if (StringUtils.isEmpty(serverAddress)) { return false; @@ -57,17 +85,76 @@ private static URI parseIpPortFromURI(String uri) { } public static NacosConfigManager chooseConfigManager(List nacosConfigManagers) { - int idx = 0; - while (idx < nacosConfigManagers.size()) { - if (checkServerConnect(nacosConfigManagers.get(idx).getServerAddr())) { - return nacosConfigManagers.get(idx); + for (NacosConfigManager nacosConfigManager : nacosConfigManagers) { + if (nacosConfigManager.isNacosServerHealth()) { + return nacosConfigManager; } - LOGGER.warn("nacos server [{}] unavailable, choose others.", nacosConfigManagers.get(idx).getServerAddr()); - idx++; + LOGGER.warn("nacos server [{}] unavailable, choose others.", nacosConfigManager.getServerAddr()); } LOGGER.warn("all nacos server unavailable, use master server."); // if all server unavailable, return master server, ensure listening configuration when service is available again. return nacosConfigManagers.get(0); } + + public static String buildUrl(String address, String uri) { + String prefix = ""; + if (!address.startsWith("http")) { + prefix = "http://"; + } + return prefix + String.format(uri, address); + } + + public static Header initHeader(String address, String userName, Properties properties) { + Header header = Header.newInstance(); + if (!StringUtils.isEmpty(userName)) { + header.addParam(NacosAuthLoginConstant.ACCESSTOKEN, getAccessToken(address, properties)); + } + return header; + } + + private static String getAccessToken(String address, Properties properties) { + if (address_token.get(address) != null + && (System.currentTimeMillis() - lastRefreshTime) < TimeUnit.SECONDS.toMillis(tokenTtl - refreshWindow)) { + return address_token.get(address); + } + HttpLoginProcessor httpLoginProcessor + = new HttpLoginProcessor(ConfigHttpClientManager.getInstance().getNacosRestTemplate()); + properties.setProperty(NacosAuthLoginConstant.SERVER, address); + LoginIdentityContext identityContext = httpLoginProcessor.getResponse(properties); + if (identityContext != null + && !StringUtils.isEmpty(identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN))) { + tokenTtl = Long.parseLong(identityContext.getParameter(NacosAuthLoginConstant.TOKENTTL)); + refreshWindow = tokenTtl / 10; + lastRefreshTime = System.currentTimeMillis(); + address_token.put(address, identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN)); + return identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN); + } + lastRefreshTime = System.currentTimeMillis(); + return ""; + } + + public static boolean checkConfigServerHealth(String serverAddr, Properties properties) { + String url = buildUrl(serverAddr, NACOS_CONFIG_HEALTH_CHECK_URI); + Header header = initHeader(serverAddr, properties.getProperty(USERNAME), properties); + try { + HttpRestResult response = getNacosRestTemplate().get(url, header, new Query(), String.class); + if (response.ok()) { + return true; + } + if (response.getCode() == HttpStatus.SC_NOT_FOUND && checkServerConnect(serverAddr)) { + return true; + } + } catch (Exception e) { + LOGGER.error("check server [{}] health failed.", serverAddr); + } + return false; + } + + public static NacosRestTemplate getNacosRestTemplate() { + if (nacosRestTemplate == null) { + nacosRestTemplate = ConfigHttpClientManager.getInstance().getNacosRestTemplate(); + } + return nacosRestTemplate; + } } diff --git a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigManager.java b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigManager.java index b0d5eb69f..efdfa7287 100644 --- a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigManager.java +++ b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigManager.java @@ -26,7 +26,11 @@ public interface NacosConfigManager extends Ordered { String getServerAddr(); - boolean checkServerConnect(); + boolean isNacosServerHealth(); void resetConfigService(); + + boolean isMasterConfigService(); + + boolean isRpcConnectHealth(); } diff --git a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceMasterManager.java b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceMasterManager.java index c56adfc8d..e762ab622 100644 --- a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceMasterManager.java +++ b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceMasterManager.java @@ -18,6 +18,9 @@ package com.huaweicloud.nacos.config.manager; import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +29,7 @@ import com.alibaba.cloud.nacos.diagnostics.analyzer.NacosConnectionFailureException; import com.alibaba.nacos.api.NacosFactory; import com.alibaba.nacos.api.config.ConfigService; +import com.huaweicloud.nacos.config.NacosConfigConst; public class NacosConfigServiceMasterManager implements NacosConfigManager { private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigServiceMasterManager.class); @@ -34,8 +38,15 @@ public class NacosConfigServiceMasterManager implements NacosConfigManager { private volatile ConfigService configService; + private final ScheduledExecutorService taskScheduler; + + private volatile boolean isServerHealth = false; + public NacosConfigServiceMasterManager(NacosConfigProperties properties) { this.properties = properties; + checkConfigServiceHealth(); + this.taskScheduler = Executors.newScheduledThreadPool(1, (t) -> new Thread(t, "Master-Config-check")); + startSchedulerTask(); } @Override @@ -55,6 +66,16 @@ public ConfigService getConfigService() { return configService; } + private void startSchedulerTask() { + long delay = properties.getMasterStandbyServerTaskDelay(); + taskScheduler.scheduleWithFixedDelay(this::checkConfigServiceHealth, delay, delay, TimeUnit.MILLISECONDS); + } + + private void checkConfigServiceHealth() { + isServerHealth = ConfigServiceManagerUtils.checkConfigServerHealth(properties.getServerAddr(), + properties.assembleMasterNacosServerProperties()); + } + @Override public String getServerAddr() { return properties.getServerAddr(); @@ -66,12 +87,26 @@ public int getOrder() { } @Override - public boolean checkServerConnect() { - return ConfigServiceManagerUtils.checkServerConnect(properties.getServerAddr()); + public boolean isNacosServerHealth() { + return isServerHealth; } @Override public void resetConfigService() { this.configService = null; + this.isServerHealth = false; + } + + @Override + public boolean isMasterConfigService() { + return true; + } + + @Override + public boolean isRpcConnectHealth() { + if (configService == null) { + return false; + } + return NacosConfigConst.STATUS_UP.equals(configService.getServerStatus()); } } diff --git a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceStandbyManager.java b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceStandbyManager.java index 23b7295f4..dc33305e2 100644 --- a/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceStandbyManager.java +++ b/spring-cloud-huawei-nacos/spring-cloud-huawei-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigServiceStandbyManager.java @@ -18,6 +18,9 @@ package com.huaweicloud.nacos.config.manager; import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +29,7 @@ import com.alibaba.cloud.nacos.diagnostics.analyzer.NacosConnectionFailureException; import com.alibaba.nacos.api.NacosFactory; import com.alibaba.nacos.api.config.ConfigService; +import com.huaweicloud.nacos.config.NacosConfigConst; public class NacosConfigServiceStandbyManager implements NacosConfigManager { private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigServiceStandbyManager.class); @@ -34,8 +38,15 @@ public class NacosConfigServiceStandbyManager implements NacosConfigManager { private volatile ConfigService configService; + private final ScheduledExecutorService taskScheduler; + + private volatile boolean isServerHealth = false; + public NacosConfigServiceStandbyManager(NacosConfigProperties properties) { this.properties = properties; + checkConfigServerHealth(); + this.taskScheduler = Executors.newScheduledThreadPool(1, (t) -> new Thread(t, "Standby-Config-Check")); + startSchedulerTask(); } @Override @@ -55,19 +66,43 @@ public ConfigService getConfigService() { return configService; } + private void startSchedulerTask() { + long delay = properties.getMasterStandbyServerTaskDelay(); + taskScheduler.scheduleWithFixedDelay(this::checkConfigServerHealth, delay, delay, TimeUnit.MILLISECONDS); + } + + private void checkConfigServerHealth() { + isServerHealth = ConfigServiceManagerUtils.checkConfigServerHealth(properties.getStandbyServerAddr(), + properties.assembleStandbyNacosServerProperties()); + } + @Override public String getServerAddr() { return properties.getStandbyServerAddr(); } @Override - public boolean checkServerConnect() { - return ConfigServiceManagerUtils.checkServerConnect(properties.getStandbyServerAddr()); + public boolean isNacosServerHealth() { + return isServerHealth; } @Override public void resetConfigService() { this.configService = null; + this.isServerHealth = false; + } + + @Override + public boolean isMasterConfigService() { + return false; + } + + @Override + public boolean isRpcConnectHealth() { + if (configService == null) { + return false; + } + return NacosConfigConst.STATUS_UP.equals(configService.getServerStatus()); } @Override