Skip to content

Commit

Permalink
fileConfig 缓存，使用 guava 替代 job 扫描，目的：提升启动速度，加快缓存失效
Browse files Browse the repository at this point in the history
  • Loading branch information
@YunaiV
YunaiV committed Sep 16, 2023
1 parent 48fd497 commit a51579a
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 103 deletions.
8 changes: 3 additions & 5 deletions ...nfra-biz/src/main/java/cn/iocoder/yudao/module/infra/dal/mysql/file/FileConfigMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@
import cn.iocoder.yudao.module.infra.controller.admin.file.vo.config.FileConfigPageReqVO;
import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileConfigDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;

import java.time.LocalDateTime;

@Mapper
public interface FileConfigMapper extends BaseMapperX<FileConfigDO> {
Expand All @@ -21,7 +18,8 @@ default PageResult<FileConfigDO> selectPage(FileConfigPageReqVO reqVO) {
.orderByDesc(FileConfigDO::getId));
}

@Select("SELECT COUNT(*) FROM infra_file_config WHERE update_time > #{maxUpdateTime}")
Long selectCountByUpdateTimeGt(LocalDateTime maxUpdateTime);
default FileConfigDO selectByMaster() {
return selectOne(FileConfigDO::getMaster, true);
}

}
117 changes: 52 additions & 65 deletions ...a-biz/src/main/java/cn/iocoder/yudao/module/infra/service/file/FileConfigServiceImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package cn.iocoder.yudao.module.infra.service.file;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.resource.ResourceUtil;
import cn.hutool.core.util.IdUtil;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.json.JsonUtils;
import cn.iocoder.yudao.framework.common.util.validation.ValidationUtils;
import cn.iocoder.yudao.framework.file.core.client.FileClient;
Expand All @@ -17,22 +15,22 @@
import cn.iocoder.yudao.module.infra.convert.file.FileConfigConvert;
import cn.iocoder.yudao.module.infra.dal.dataobject.file.FileConfigDO;
import cn.iocoder.yudao.module.infra.dal.mysql.file.FileConfigMapper;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.validation.annotation.Validated;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.validation.Validator;
import java.time.LocalDateTime;
import java.util.List;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.Objects;

import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.framework.common.util.cache.CacheUtils.buildAsyncReloadingCache;
import static cn.iocoder.yudao.module.infra.enums.ErrorCodeConstants.FILE_CONFIG_DELETE_FAIL_MASTER;
import static cn.iocoder.yudao.module.infra.enums.ErrorCodeConstants.FILE_CONFIG_NOT_EXISTS;

Expand All @@ -46,73 +44,42 @@
@Slf4j
public class FileConfigServiceImpl implements FileConfigService {

@Resource
private FileClientFactory fileClientFactory;
private static final Long CACHE_MASTER_ID = 0L;

/**
* 文件配置的缓存
*/
@Getter
private List<FileConfigDO> fileConfigCache;
/**
* Master FileClient 对象，有且仅有一个，即 {@link FileConfigDO#getMaster()} 对应的
* {@link FileClient} 缓存，通过它异步刷新 fileClientFactory
*/
@Getter
private FileClient masterFileClient;
private final LoadingCache<Long, FileClient> clientCache = buildAsyncReloadingCache(Duration.ofSeconds(10L),
new CacheLoader<Long, FileClient>() {

@Override
public FileClient load(Long id) {
FileConfigDO config = Objects.equals(CACHE_MASTER_ID, id) ?
fileConfigMapper.selectByMaster() : fileConfigMapper.selectById(id);
if (config != null) {
fileClientFactory.createOrUpdateFileClient(id, config.getStorage(), config.getConfig());
}
return fileClientFactory.getFileClient(id);
}

});

@Resource
private FileClientFactory fileClientFactory;

@Resource
private FileConfigMapper fileConfigMapper;

@Resource
private Validator validator;

@PostConstruct
public void initLocalCache() {
// 第一步：查询数据
List<FileConfigDO> configs = fileConfigMapper.selectList();
log.info("[initLocalCache][缓存文件配置，数量为:{}]", configs.size());

// 第二步：构建缓存：创建或更新文件 Client
configs.forEach(config -> {
fileClientFactory.createOrUpdateFileClient(config.getId(), config.getStorage(), config.getConfig());
// 如果是 master，进行设置
if (Boolean.TRUE.equals(config.getMaster())) {
masterFileClient = fileClientFactory.getFileClient(config.getId());
}
});
this.fileConfigCache = configs;
}

/**
* 通过定时任务轮询，刷新缓存
*
* 目的：多节点部署时，通过轮询”通知“所有节点，进行刷新
*/
@Scheduled(initialDelay = 60, fixedRate = 60, timeUnit = TimeUnit.SECONDS)
public void refreshLocalCache() {
// 情况一：如果缓存里没有数据，则直接刷新缓存
if (CollUtil.isEmpty(fileConfigCache)) {
initLocalCache();
return;
}

// 情况二，如果缓存里数据，则通过 updateTime 判断是否有数据变更，有变更则刷新缓存
LocalDateTime maxTime = CollectionUtils.getMaxValue(fileConfigCache, FileConfigDO::getUpdateTime);
if (fileConfigMapper.selectCountByUpdateTimeGt(maxTime) > 0) {
initLocalCache();
}
}

@Override
public Long createFileConfig(FileConfigCreateReqVO createReqVO) {
// 插入
FileConfigDO fileConfig = FileConfigConvert.INSTANCE.convert(createReqVO)
.setConfig(parseClientConfig(createReqVO.getStorage(), createReqVO.getConfig()))
.setMaster(false); // 默认非 master
fileConfigMapper.insert(fileConfig);

// 刷新缓存
initLocalCache();
return fileConfig.getId();
}

Expand All @@ -125,8 +92,8 @@ public void updateFileConfig(FileConfigUpdateReqVO updateReqVO) {
.setConfig(parseClientConfig(config.getStorage(), updateReqVO.getConfig()));
fileConfigMapper.updateById(updateObj);

// 刷新缓存
initLocalCache();
// 清空缓存
clearCache(config.getId(), null);
}

@Override
Expand All @@ -139,8 +106,8 @@ public void updateFileConfigMaster(Long id) {
// 更新
fileConfigMapper.updateById(new FileConfigDO().setId(id).setMaster(true));

// 刷新缓存
initLocalCache();
// 清空缓存
clearCache(null, true);
}

private FileClientConfig parseClientConfig(Integer storage, Map<String, Object> config) {
Expand All @@ -164,8 +131,23 @@ public void deleteFileConfig(Long id) {
// 删除
fileConfigMapper.deleteById(id);

// 刷新缓存
initLocalCache();
// 清空缓存
clearCache(id, null);
}

/**
* 清空指定文件配置
*
* @param id 配置编号
* @param master 是否主配置
*/
private void clearCache(Long id, Boolean master) {
if (id != null) {
clientCache.invalidate(id);
}
if (Boolean.TRUE.equals(master)) {
clientCache.invalidate(CACHE_MASTER_ID);
}
}

private FileConfigDO validateFileConfigExists(Long id) {
Expand All @@ -192,12 +174,17 @@ public String testFileConfig(Long id) throws Exception {
validateFileConfigExists(id);
// 上传文件
byte[] content = ResourceUtil.readBytes("file/erweima.jpg");
return fileClientFactory.getFileClient(id).upload(content, IdUtil.fastSimpleUUID() + ".jpg", "image/jpeg");
return getFileClient(id).upload(content, IdUtil.fastSimpleUUID() + ".jpg", "image/jpeg");
}

@Override
public FileClient getFileClient(Long id) {
return fileClientFactory.getFileClient(id);
return clientCache.getUnchecked(id);
}

@Override
public FileClient getMasterFileClient() {
return clientCache.getUnchecked(CACHE_MASTER_ID);
}

}
33 changes: 8 additions & 25 deletions ...z/src/test/java/cn/iocoder/yudao/module/infra/service/file/FileConfigServiceImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,31 +59,6 @@ public class FileConfigServiceImplTest extends BaseDbUnitTest {
@MockBean
private FileClientFactory fileClientFactory;

@Test
public void testInitLocalCache() {
// mock 数据
FileConfigDO configDO1 = randomFileConfigDO().setId(1L).setMaster(true);
fileConfigMapper.insert(configDO1);
FileConfigDO configDO2 = randomFileConfigDO().setId(2L).setMaster(false);
fileConfigMapper.insert(configDO2);
// mock fileClientFactory 获得 master
FileClient masterFileClient = mock(FileClient.class);
when(fileClientFactory.getFileClient(eq(1L))).thenReturn(masterFileClient);

// 调用
fileConfigService.initLocalCache();
// 断言 fileClientFactory 调用
verify(fileClientFactory).createOrUpdateFileClient(eq(1L),
eq(configDO1.getStorage()), eq(configDO1.getConfig()));
verify(fileClientFactory).createOrUpdateFileClient(eq(2L),
eq(configDO2.getStorage()), eq(configDO2.getConfig()));
assertSame(masterFileClient, fileConfigService.getMasterFileClient());
// 断言 fileConfigCache 缓存
assertEquals(2, fileConfigService.getFileConfigCache().size());
assertEquals(configDO1, fileConfigService.getFileConfigCache().get(0));
assertEquals(configDO2, fileConfigService.getFileConfigCache().get(1));
}

@Test
public void testCreateFileConfig_success() {
// 准备参数
Expand All @@ -102,6 +77,8 @@ public void testCreateFileConfig_success() {
assertFalse(fileConfig.getMaster());
assertEquals("/yunai", ((LocalFileClientConfig) fileConfig.getConfig()).getBasePath());
assertEquals("https://www.iocoder.cn", ((LocalFileClientConfig) fileConfig.getConfig()).getDomain());
// 验证 cache
assertNull(fileConfigService.getClientCache().getIfPresent(fileConfigId));
}

@Test
Expand All @@ -125,6 +102,8 @@ public void testUpdateFileConfig_success() {
assertPojoEquals(reqVO, fileConfig, "config");
assertEquals("/yunai2", ((LocalFileClientConfig) fileConfig.getConfig()).getBasePath());
assertEquals("https://doc.iocoder.cn", ((LocalFileClientConfig) fileConfig.getConfig()).getDomain());
// 验证 cache
assertNull(fileConfigService.getClientCache().getIfPresent(fileConfig.getId()));
}

@Test
Expand All @@ -149,6 +128,8 @@ public void testUpdateFileConfigMaster_success() {
// 断言数据
assertTrue(fileConfigMapper.selectById(dbFileConfig.getId()).getMaster());
assertFalse(fileConfigMapper.selectById(masterFileConfig.getId()).getMaster());
// 验证 cache
assertNull(fileConfigService.getClientCache().getIfPresent(0L));
}

@Test
Expand All @@ -169,6 +150,8 @@ public void testDeleteFileConfig_success() {
fileConfigService.deleteFileConfig(id);
// 校验数据不存在了
assertNull(fileConfigMapper.selectById(id));
// 验证 cache
assertNull(fileConfigService.getClientCache().getIfPresent(id));
}

@Test
Expand Down
1 change: 1 addition & 0 deletions ...fra-biz/src/test/java/cn/iocoder/yudao/module/infra/service/file/FileServiceImplTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,4 +139,5 @@ public void testGetFileContent() throws Exception {
// 断言
assertSame(result, content);
}

}
12 changes: 4 additions & 8 deletions ...m-biz/src/main/java/cn/iocoder/yudao/module/system/service/sms/SmsChannelServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import cn.hutool.core.util.StrUtil;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.cache.CacheUtils;
import cn.iocoder.yudao.framework.sms.core.client.SmsClient;
import cn.iocoder.yudao.framework.sms.core.client.SmsClientFactory;
import cn.iocoder.yudao.framework.sms.core.property.SmsChannelProperties;
Expand All @@ -15,7 +14,6 @@
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import lombok.Getter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;

Expand All @@ -24,6 +22,7 @@
import java.util.List;

import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.framework.common.util.cache.CacheUtils.buildAsyncReloadingCache;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SMS_CHANNEL_HAS_CHILDREN;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SMS_CHANNEL_NOT_EXISTS;

Expand All @@ -40,7 +39,7 @@ public class SmsChannelServiceImpl implements SmsChannelService {
* {@link SmsClient} 缓存，通过它异步刷新 smsClientFactory
*/
@Getter
private final LoadingCache<Long, SmsClient> idClientCache = CacheUtils.buildAsyncReloadingCache(Duration.ofSeconds(10L),
private final LoadingCache<Long, SmsClient> idClientCache = buildAsyncReloadingCache(Duration.ofSeconds(10L),
new CacheLoader<Long, SmsClient>() {

@Override
Expand All @@ -60,7 +59,7 @@ public SmsClient load(Long id) {
* {@link SmsClient} 缓存，通过它异步刷新 smsClientFactory
*/
@Getter
private final LoadingCache<String, SmsClient> codeClientCache = CacheUtils.buildAsyncReloadingCache(Duration.ofSeconds(60L),
private final LoadingCache<String, SmsClient> codeClientCache = buildAsyncReloadingCache(Duration.ofSeconds(60L),
new CacheLoader<String, SmsClient>() {

@Override
Expand All @@ -87,12 +86,8 @@ public SmsClient load(String code) {

@Override
public Long createSmsChannel(SmsChannelCreateReqVO createReqVO) {
// 插入
SmsChannelDO channel = SmsChannelConvert.INSTANCE.convert(createReqVO);
smsChannelMapper.insert(channel);

// 清空缓存
clearCache(channel.getId(), null);
return channel.getId();
}

Expand Down Expand Up @@ -127,6 +122,7 @@ public void deleteSmsChannel(Long id) {
* 清空指定渠道编号的缓存
*
* @param id 渠道编号
* @param code 渠道编码
*/
private void clearCache(Long id, String code) {
idClientCache.invalidate(id);
Expand Down

0 comments on commit a51579a

Please sign in to comment.