Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#1427] Optimized dual-engine config center health check logic #1439

Merged
merged 2 commits into from
Feb 22, 2025
Merged

[#1427] Optimized dual-engine config center health check logic #1439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f4d0dd4
[#1427] Optimized dual-engine config center health check logic
chengyouling Feb 10, 2025
fee2841
修复ci问题
chengyouling Feb 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...-nacos-config/src/main/java/com/alibaba/cloud/nacos/configdata/NacosConfigDataLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,11 @@ public ConfigData doLoad(ConfigDataLoaderContext context,

private ConfigService getConfigService(ConfigDataLoaderContext context) {
NacosConfigServiceMasterManager configServiceMaster = getBean(context, NacosConfigServiceMasterManager.class);
if (configServiceMaster != null && configServiceMaster.checkServerConnect()) {
if (configServiceMaster != null && configServiceMaster.isNacosServerHealth()) {
return configServiceMaster.getConfigService();
}
NacosConfigServiceStandbyManager configServiceStandby = getBean(context, NacosConfigServiceStandbyManager.class);
if (configServiceStandby != null && configServiceStandby.checkServerConnect()) {
if (configServiceStandby != null && configServiceStandby.isNacosServerHealth()) {
return configServiceStandby.getConfigService();
}
if (configServiceMaster != null) {
Expand Down
16 changes: 13 additions & 3 deletions ...wei-nacos-config/src/main/java/com/alibaba/cloud/nacos/refresh/NacosContextRefresher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,8 @@ private void startSchedulerTask() {

private void masterStandbyServerCheck() {
synchronized (NacosContextRefresher.class) {
if (!NacosConfigConst.STATUS_UP.equals(currentConfigServiceManager.getConfigService().getServerStatus())) {
// both check interface invoking and GRPC connect for fail count.
if (!currentConfigServiceManager.isNacosServerHealth() || !currentConfigServiceManager.isRpcConnectHealth()) {
if (failCount == 2) {
reRegisterNacosListeners();
failCount = 0;
Expand All @@ -157,13 +158,22 @@ private void masterStandbyServerCheck() {

// if current server is not master, check whether the master status is healthy, if status UP, use master server.
if (isRetryMasterServer
&& !currentConfigServiceManager.getServerAddr().equals(nacosConfigManagers.get(0).getServerAddr())
&& nacosConfigManagers.get(0).checkServerConnect()) {
&& !currentConfigServiceManager.isMasterConfigService()
&& getMasterConfigService().isNacosServerHealth()) {
reRegisterNacosListeners();
}
}
}

private NacosConfigManager getMasterConfigService() {
for (NacosConfigManager configManager : nacosConfigManagers) {
if (configManager.isMasterConfigService()) {
return configManager;
}
}
return nacosConfigManagers.get(0);
}

private void reRegisterNacosListeners() {
shutDownConfigService(currentConfigServiceManager);
clearConfigListener();
Expand Down
107 changes: 35 additions & 72 deletions ...g/src/main/java/com/huaweicloud/nacos/config/client/NacosPropertiesFuzzyQueryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,23 @@

package com.huaweicloud.nacos.config.client;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;

import com.alibaba.cloud.nacos.NacosConfigProperties;
import com.alibaba.nacos.client.auth.impl.NacosAuthLoginConstant;
import com.alibaba.nacos.client.auth.impl.process.HttpLoginProcessor;
import com.alibaba.nacos.client.config.impl.ConfigHttpClientManager;
import com.alibaba.nacos.common.http.HttpRestResult;
import com.alibaba.nacos.common.http.client.NacosRestTemplate;
import com.alibaba.nacos.common.http.param.Header;
import com.alibaba.nacos.common.http.param.Query;
import com.alibaba.nacos.plugin.auth.api.LoginIdentityContext;
import com.huaweicloud.nacos.config.NacosConfigConst;
import com.huaweicloud.nacos.config.manager.ConfigServiceManagerUtils;

Expand All @@ -45,19 +42,11 @@ public class NacosPropertiesFuzzyQueryService {

private static final String NACOS_CONFIG_QUERY_URI = "%s/nacos/v1/cs/configs";

private NacosConfigProperties configProperties;

private Properties properties;

private final Map<String, String> address_token = new ConcurrentHashMap<>();

private long tokenTtl;

private long lastRefreshTime;
private static final NacosPropertiesFuzzyQueryService INSTANCE = new NacosPropertiesFuzzyQueryService();

private long refreshWindow;
private NacosConfigProperties configProperties;

private static final NacosPropertiesFuzzyQueryService INSTANCE = new NacosPropertiesFuzzyQueryService();
private final Map<String, Properties> addressPropertiesMap = new HashMap<>();

private NacosPropertiesFuzzyQueryService() {

Expand All @@ -68,16 +57,24 @@ public static NacosPropertiesFuzzyQueryService getInstance() {
}

public List<PropertyConfigItem> loadRouterProperties() {
try {
String address = chooseAddress();
NacosRestTemplate nacosRestTemplate = ConfigHttpClientManager.getInstance().getNacosRestTemplate();
HttpRestResult<PropertiePageQueryResult> response = nacosRestTemplate.get(buildUrl(address), initHeader(address),
buildQuery(), PropertiePageQueryResult.class);
return response.getData().getPageItems();
} catch (Exception e) {
LOGGER.error("load router properties failed!", e);
return Collections.emptyList();
buildAddressPropertiesMap();
List<String> addresses = getAddresses();
for (String address : addresses) {
try {
NacosRestTemplate nacosRestTemplate = ConfigServiceManagerUtils.getNacosRestTemplate();
String url = ConfigServiceManagerUtils.buildUrl(address, NACOS_CONFIG_QUERY_URI);
Header header = ConfigServiceManagerUtils.initHeader(address, configProperties.getUsername(),
addressPropertiesMap.get(address));
HttpRestResult<PropertiePageQueryResult> response = nacosRestTemplate.get(url, header, buildQuery(),
PropertiePageQueryResult.class);
if (response.getData() != null && !CollectionUtils.isEmpty(response.getData().getPageItems())) {
return response.getData().getPageItems();
}
} catch (Exception e) {
LOGGER.error("load router properties failed!", e);
}
}
return Collections.emptyList();
}

private Query buildQuery() {
Expand All @@ -93,60 +90,26 @@ private Query buildQuery() {
return query;
}

private Header initHeader(String address) {
Header header = Header.newInstance();
if (!StringUtils.isEmpty(configProperties.getUsername())) {
header.addParam(NacosAuthLoginConstant.ACCESSTOKEN, getAccessToken(address));
}
return header;
}

private String buildUrl(String address) {
String prefix = "";
if (!configProperties.getServerAddr().startsWith("http")) {
prefix = "http://";
}
return prefix + String.format(NACOS_CONFIG_QUERY_URI, address);
}

private String getNamespace() {
return StringUtils.isEmpty(configProperties.getNamespace()) ? "" : configProperties.getNamespace();
}

private String chooseAddress() {
properties = configProperties.assembleMasterNacosServerProperties();
if (!configProperties.isMasterStandbyEnabled()) {
return configProperties.getServerAddr();
}
if (ConfigServiceManagerUtils.checkServerConnect(configProperties.getServerAddr())) {
return configProperties.getServerAddr();
private List<String> getAddresses() {
List<String> addresses = new ArrayList<>();
addresses.add(configProperties.getServerAddr());
if (configProperties.isMasterStandbyEnabled() && !StringUtils.isEmpty(configProperties.getStandbyServerAddr())) {
addresses.add(configProperties.getStandbyServerAddr());
}
if (ConfigServiceManagerUtils.checkServerConnect(configProperties.getStandbyServerAddr())) {
properties = configProperties.assembleStandbyNacosServerProperties();
return configProperties.getStandbyServerAddr();
}
return configProperties.getServerAddr();
return addresses;
}

private String getAccessToken(String address) {
if (address_token.get(address) != null
&& (System.currentTimeMillis() - lastRefreshTime) < TimeUnit.SECONDS.toMillis(tokenTtl - refreshWindow)) {
return address_token.get(address);
}
HttpLoginProcessor httpLoginProcessor
= new HttpLoginProcessor(ConfigHttpClientManager.getInstance().getNacosRestTemplate());
properties.setProperty(NacosAuthLoginConstant.SERVER, address);
LoginIdentityContext identityContext = httpLoginProcessor.getResponse(properties);
if (identityContext != null
&& !StringUtils.isEmpty(identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN))) {
tokenTtl = Long.parseLong(identityContext.getParameter(NacosAuthLoginConstant.TOKENTTL));
refreshWindow = tokenTtl / 10;
lastRefreshTime = System.currentTimeMillis();
address_token.put(address, identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN));
return identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN);
private void buildAddressPropertiesMap() {
Properties masterProperties = configProperties.assembleMasterNacosServerProperties();
addressPropertiesMap.put(configProperties.getServerAddr(), masterProperties);
if (!StringUtils.isEmpty(configProperties.getStandbyServerAddr())) {
Properties standbyProperties = configProperties.assembleStandbyNacosServerProperties();
addressPropertiesMap.put(configProperties.getStandbyServerAddr(), standbyProperties);
}
lastRefreshTime = System.currentTimeMillis();
return "";
}

public void setConfigProperties(NacosConfigProperties nacosConfigProperties) {
Expand Down
99 changes: 93 additions & 6 deletions ...-config/src/main/java/com/huaweicloud/nacos/config/manager/ConfigServiceManagerUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,48 @@

package com.huaweicloud.nacos.config.manager;

import static com.alibaba.nacos.api.PropertyKeyConst.USERNAME;

import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.nacos.client.auth.impl.NacosAuthLoginConstant;
import com.alibaba.nacos.client.auth.impl.process.HttpLoginProcessor;
import com.alibaba.nacos.client.config.impl.ConfigHttpClientManager;
import com.alibaba.nacos.common.http.HttpRestResult;
import com.alibaba.nacos.common.http.client.NacosRestTemplate;
import com.alibaba.nacos.common.http.param.Header;
import com.alibaba.nacos.common.http.param.Query;
import com.alibaba.nacos.plugin.auth.api.LoginIdentityContext;

public class ConfigServiceManagerUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(ConfigServiceManagerUtils.class);

private static final Map<String, String> address_token = new ConcurrentHashMap<>();

private static final String NACOS_CONFIG_HEALTH_CHECK_URI = "%s/nacos/v1/console/health/config/readiness";

private static long tokenTtl;

private static long lastRefreshTime;

private static long refreshWindow;

private static volatile NacosRestTemplate nacosRestTemplate;

public static boolean checkServerConnect(String serverAddress) {
if (StringUtils.isEmpty(serverAddress)) {
return false;
Expand All @@ -57,17 +85,76 @@ private static URI parseIpPortFromURI(String uri) {
}

public static NacosConfigManager chooseConfigManager(List<NacosConfigManager> nacosConfigManagers) {
int idx = 0;
while (idx < nacosConfigManagers.size()) {
if (checkServerConnect(nacosConfigManagers.get(idx).getServerAddr())) {
return nacosConfigManagers.get(idx);
for (NacosConfigManager nacosConfigManager : nacosConfigManagers) {
if (nacosConfigManager.isNacosServerHealth()) {
return nacosConfigManager;
}
LOGGER.warn("nacos server [{}] unavailable, choose others.", nacosConfigManagers.get(idx).getServerAddr());
idx++;
LOGGER.warn("nacos server [{}] unavailable, choose others.", nacosConfigManager.getServerAddr());
}
LOGGER.warn("all nacos server unavailable, use master server.");

// if all server unavailable, return master server, ensure listening configuration when service is available again.
return nacosConfigManagers.get(0);
}

public static String buildUrl(String address, String uri) {
String prefix = "";
if (!address.startsWith("http")) {
prefix = "http://";
}
return prefix + String.format(uri, address);
}

public static Header initHeader(String address, String userName, Properties properties) {
Header header = Header.newInstance();
if (!StringUtils.isEmpty(userName)) {
header.addParam(NacosAuthLoginConstant.ACCESSTOKEN, getAccessToken(address, properties));
}
return header;
}

private static String getAccessToken(String address, Properties properties) {
if (address_token.get(address) != null
&& (System.currentTimeMillis() - lastRefreshTime) < TimeUnit.SECONDS.toMillis(tokenTtl - refreshWindow)) {
return address_token.get(address);
}
HttpLoginProcessor httpLoginProcessor
= new HttpLoginProcessor(ConfigHttpClientManager.getInstance().getNacosRestTemplate());
properties.setProperty(NacosAuthLoginConstant.SERVER, address);
LoginIdentityContext identityContext = httpLoginProcessor.getResponse(properties);
if (identityContext != null
&& !StringUtils.isEmpty(identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN))) {
tokenTtl = Long.parseLong(identityContext.getParameter(NacosAuthLoginConstant.TOKENTTL));
refreshWindow = tokenTtl / 10;
lastRefreshTime = System.currentTimeMillis();
address_token.put(address, identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN));
return identityContext.getParameter(NacosAuthLoginConstant.ACCESSTOKEN);
}
lastRefreshTime = System.currentTimeMillis();
return "";
}

public static boolean checkConfigServerHealth(String serverAddr, Properties properties) {
String url = buildUrl(serverAddr, NACOS_CONFIG_HEALTH_CHECK_URI);
Header header = initHeader(serverAddr, properties.getProperty(USERNAME), properties);
try {
HttpRestResult<String> response = getNacosRestTemplate().get(url, header, new Query(), String.class);
if (response.ok()) {
return true;
}
if (response.getCode() == HttpStatus.SC_NOT_FOUND && checkServerConnect(serverAddr)) {
return true;
}
} catch (Exception e) {
LOGGER.error("check server [{}] health failed.", serverAddr);
}
return false;
}

public static NacosRestTemplate getNacosRestTemplate() {
if (nacosRestTemplate == null) {
nacosRestTemplate = ConfigHttpClientManager.getInstance().getNacosRestTemplate();
}
return nacosRestTemplate;
}
}
6 changes: 5 additions & 1 deletion ...i-nacos-config/src/main/java/com/huaweicloud/nacos/config/manager/NacosConfigManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ public interface NacosConfigManager extends Ordered {

String getServerAddr();

boolean checkServerConnect();
boolean isNacosServerHealth();

void resetConfigService();

boolean isMasterConfigService();

boolean isRpcConnectHealth();
}
Loading
Loading