Skip to content

Commit

Permalink
[issue #353] support read yaml file from config center (#352)
Browse files Browse the repository at this point in the history
* support read yaml file from config center

* add licence

* update demo

* delete testing code

* modify as comment
  • Loading branch information
GuoYL123 authored Feb 4, 2021
1 parent b39bca1 commit 78c9821
Show file tree
Hide file tree
Showing 14 changed files with 268 additions and 91 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,19 @@ public class ServiceCombConfigProperties {
@Value("${spring.cloud.servicecomb.config.enableLongPolling:true}")
private boolean enableLongPolling;

@Value("${spring.cloud.servicecomb.config.fileSource:}")
private String fileSource;

private Watch watch = new Watch();

public String getFileSource() {
return fileSource;
}

public void setFileSource(String fileSource) {
this.fileSource = fileSource;
}

public String getDiscoveryAddress() {
return discoveryAddress;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.huaweicloud.common.transport.HttpTransport;
import com.huaweicloud.common.transport.Response;
import com.huaweicloud.config.ServiceCombConfigProperties;
import com.huaweicloud.config.client.kie.ConfigCenterFileProcessor;

import java.io.IOException;
import java.net.URLEncoder;
Expand All @@ -42,9 +43,11 @@ public class ConfigCenterClient extends ServiceCombConfigClient {

private static final Logger LOGGER = LoggerFactory.getLogger(ConfigCenterClient.class);

private final ConfigCenterFileProcessor processor = new ConfigCenterFileProcessor();

public ConfigCenterClient(String urls,
HttpTransport httpTransport) {
super(urls, httpTransport);
HttpTransport httpTransport, String fileSource) {
super(urls, httpTransport, fileSource);
}

public Map<String, Object> loadAll(ServiceCombConfigProperties serviceCombConfigProperties,
Expand Down Expand Up @@ -73,19 +76,7 @@ public Map<String, Object> loadAll(ServiceCombConfigProperties serviceCombConfig
if (allConfigMap.get(ConfigConstants.REVISION) != null) {
revision = (String) allConfigMap.get(ConfigConstants.REVISION).get("version");
}
if (allConfigMap.get(ConfigConstants.APPLICATION_CONFIG) != null) {
result.putAll(allConfigMap.get(ConfigConstants.APPLICATION_CONFIG));
}
if (dimensionsInfo.contains(ConfigConstants.DEFAULT_SERVICE_SEPARATOR)
&& allConfigMap.get(dimensionsInfo
.substring(0, dimensionsInfo.indexOf(ConfigConstants.DEFAULT_SERVICE_SEPARATOR)))
!= null) {
result.putAll(allConfigMap.get(dimensionsInfo
.substring(0, dimensionsInfo.indexOf(ConfigConstants.DEFAULT_SERVICE_SEPARATOR))));
}
if (allConfigMap.get(dimensionsInfo) != null) {
result.putAll(allConfigMap.get(dimensionsInfo));
}
return mergeConfig(allConfigMap, dimensionsInfo);
}
return result;
} else if (response.getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
Expand All @@ -112,6 +103,27 @@ public Map<String, Object> loadAll(ServiceCombConfigProperties serviceCombConfig
}
}

private Map<String, Object> mergeConfig(Map<String, Map<String, Object>> allConfigMap, String dimensionsInfo) {
Map<String, Object> result = new HashMap<>();
Map<String, Object> kvConfig = new HashMap<>();
Map<String, Object> fileConfig = new HashMap<>();
if (allConfigMap.get(ConfigConstants.APPLICATION_CONFIG) != null) {
filterConfig(allConfigMap.get(ConfigConstants.APPLICATION_CONFIG), kvConfig, fileConfig);
}
String serviceKey = dimensionsInfo
.substring(0, dimensionsInfo.indexOf(ConfigConstants.DEFAULT_SERVICE_SEPARATOR));
if (dimensionsInfo.contains(ConfigConstants.DEFAULT_SERVICE_SEPARATOR)
&& allConfigMap.get(serviceKey) != null) {
filterConfig(allConfigMap.get(serviceKey), kvConfig, fileConfig);
}
if (allConfigMap.get(dimensionsInfo) != null) {
filterConfig(allConfigMap.get(dimensionsInfo), kvConfig, fileConfig);
}
result.putAll(processor.process(fileConfig));
result.putAll(kvConfig);
return result;
}


private String spliceDimensionsInfo(ServiceCombConfigProperties serviceCombConfigProperties) {
String result =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,23 @@
package com.huaweicloud.config.client;

import com.huaweicloud.common.exception.RemoteOperationException;
import com.huaweicloud.common.exception.ServiceCombRuntimeException;
import com.huaweicloud.common.transport.HttpTransport;
import com.huaweicloud.common.transport.Response;
import com.huaweicloud.config.ServiceCombConfigProperties;
import com.huaweicloud.config.client.kie.KieDefaultProcessor;
import com.huaweicloud.config.model.KVDoc;
import com.huaweicloud.config.model.KVResponse;
import com.huaweicloud.config.model.ValueType;
import java.io.StringReader;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.http.HttpStatus;
import org.apache.servicecomb.foundation.common.utils.JsonUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.util.StringUtils;

/**
Expand All @@ -49,12 +44,13 @@ public class KieClient extends ServiceCombConfigClient {

private static final Logger LOGGER = LoggerFactory.getLogger(KieClient.class);

private final KieDefaultProcessor processor = new KieDefaultProcessor();

private AtomicBoolean isFirst = new AtomicBoolean(true);
private final AtomicBoolean isFirst = new AtomicBoolean(true);

public KieClient(String urls,
HttpTransport httpTransport) {
super(urls, httpTransport);
HttpTransport httpTransport, String fileSource) {
super(urls, httpTransport, fileSource);
}

public Map<String, Object> loadAll(ServiceCombConfigProperties serviceCombConfigProperties,
Expand Down Expand Up @@ -144,68 +140,14 @@ private Map<String, Object> getConfigByLabel(
}
//kv is priority
for (KVDoc kvDoc : appList) {
resultMap.putAll(processValueType(kvDoc));
resultMap.putAll(processor.process(kvDoc));
}
for (KVDoc kvDoc : serviceList) {
resultMap.putAll(processValueType(kvDoc));
resultMap.putAll(processor.process(kvDoc));
}
for (KVDoc kvDoc : versionList) {
resultMap.putAll(processValueType(kvDoc));
resultMap.putAll(processor.process(kvDoc));
}
return resultMap;
}


private Map<String, Object> processValueType(KVDoc kvDoc) {
ValueType vtype;
try {
vtype = ValueType.valueOf(kvDoc.getValueType());
} catch (IllegalArgumentException e) {
throw new ServiceCombRuntimeException("value type not support");
}
Properties properties = new Properties();
Map<String, Object> kvMap = new HashMap<>();
try {
switch (vtype) {
case yml:
case yaml:
YamlPropertiesFactoryBean yamlFactory = new YamlPropertiesFactoryBean();
yamlFactory.setResources(new ByteArrayResource(kvDoc.getValue().getBytes()));
return toMap(kvDoc.getKey(), yamlFactory.getObject());
case properties:
properties.load(new StringReader(kvDoc.getValue()));
return toMap(kvDoc.getKey(), properties);
case text:
case string:
default:
kvMap.put(kvDoc.getKey(), kvDoc.getValue());
return kvMap;
}
} catch (Exception e) {
LOGGER.error("read config failed");
}
return Collections.emptyMap();
}


private Map<String, Object> toMap(String prefix, Properties properties) {
if (properties == null) {
return Collections.emptyMap();
}
Map<String, Object> result = new HashMap<>();
Enumeration<String> keys = (Enumeration<String>) properties.propertyNames();
while (keys.hasMoreElements()) {
String key = keys.nextElement();
Object value = properties.getProperty(key);
if (!StringUtils.isEmpty(prefix)) {
key = prefix + "." + key;
}
if (value != null) {
result.put(key, value);
} else {
result.put(key, null);
}
}
return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,15 @@
import com.huaweicloud.common.transport.URLConfig;
import com.huaweicloud.common.util.URLUtil;
import com.huaweicloud.config.ServiceCombConfigProperties;

import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.lang3.StringUtils;

import com.huaweicloud.common.exception.RemoteOperationException;
import com.huaweicloud.common.transport.HttpTransport;

Expand All @@ -34,14 +42,30 @@ public abstract class ServiceCombConfigClient {

protected URLConfig configCenterConfig = new URLConfig();

protected List<String> fileSources = new ArrayList<>();

protected String revision = "0";

public ServiceCombConfigClient(String urls, HttpTransport httpTransport) {
public ServiceCombConfigClient(String urls, HttpTransport httpTransport, String fileSource) {
this.httpTransport = httpTransport;
configCenterConfig.addUrl(URLUtil.getEnvConfigUrl());
if (configCenterConfig.isEmpty()) {
configCenterConfig.addUrl(URLUtil.dealMultiUrl(urls));
}
if (StringUtils.isNotEmpty(fileSource)) {
fileSources = Arrays.asList(fileSource.split(","));
}
}

protected void filterConfig(Map<String, Object> rawConfig, Map<String, Object> kvConfig,
Map<String, Object> fileConfig) {
for (Entry<String, Object> entry : rawConfig.entrySet()) {
if (fileSources.contains(entry.getKey())) {
fileConfig.put(entry.getKey(), entry.getValue());
} else {
kvConfig.put(entry.getKey(), entry.getValue());
}
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,10 @@ public ServiceCombConfigClient createServiceCombConfigClient() {
serviceCombRBACProperties, serviceCombConfigProperties.getDiscoveryAddress(),
DealHeaderUtil.LONG_POLLING_SOCKET_TIMEOUT);
if ("kie".equalsIgnoreCase(serviceCombConfigProperties.getServerType())) {
return new KieClient(serviceCombConfigProperties.getServerAddr(), httpTransport);
return new KieClient(serviceCombConfigProperties.getServerAddr(), httpTransport,
serviceCombConfigProperties.getFileSource());
}
return new ConfigCenterClient(serviceCombConfigProperties.getServerAddr(), httpTransport);
return new ConfigCenterClient(serviceCombConfigProperties.getServerAddr(), httpTransport,
serviceCombConfigProperties.getFileSource());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.huaweicloud.config.client.kie;

import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
import org.springframework.core.io.ByteArrayResource;

public class ConfigCenterFileProcessor extends ConfigValueProcessor<Map<String, Object>> {
@Override
public Map<String, Object> process(Map<String, Object> source) {
Map<String, Object> result = new HashMap<>();
for (Entry<String, Object> entry : source.entrySet()) {
YamlPropertiesFactoryBean yamlFactory = new YamlPropertiesFactoryBean();
yamlFactory.setResources(new ByteArrayResource(((String) entry.getValue()).getBytes()));
result.putAll(toMap("", yamlFactory.getObject()));
}
return result;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.huaweicloud.config.client.kie;

import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

import org.springframework.util.StringUtils;

public abstract class ConfigValueProcessor<T> {

abstract Map<String, Object> process(T source);

protected Map<String, Object> toMap(String prefix, Properties properties) {
if (properties == null) {
return Collections.emptyMap();
}
Map<String, Object> result = new HashMap<>();
Enumeration<String> keys = (Enumeration<String>) properties.propertyNames();
while (keys.hasMoreElements()) {
String key = keys.nextElement();
Object value = properties.getProperty(key);
if (!StringUtils.isEmpty(prefix)) {
key = prefix + "." + key;
}
if (value != null) {
result.put(key, value);
} else {
result.put(key, null);
}
}
return result;
}
}
Loading

0 comments on commit 78c9821

Please sign in to comment.