Skip to content

♻️ [REFACTOR#45] NewsAPI 관련 데이터 적재 : try-catch 를 통한 Spring Context 종료 방지 #47

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kimjy0117 merged 2 commits into from
Apr 8, 2025
Merged

♻️ [REFACTOR#45] NewsAPI 관련 데이터 적재 : try-catch 를 통한 Spring Context 종료 방지 #47

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ffac1c1
:recycle: Refactor: 뉴스 데이터 DB 저장 관련 에러 핸들링 추가
Oyisbe Apr 7, 2025
01e592f
Merge pull request #46 from SKU-GlobalTimes/refactor/45
kimjy0117 Apr 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 83 additions & 58 deletions src/main/java/com/example/globalTimes_be/externalApi/service/NewsApiService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,24 +55,37 @@ public NewsApiService(RestTemplate restTemplate, ArticleRepository articleReposi

@PostConstruct
public void init() {
fetchTopHeadlines(true); // 뉴스 초기 적재
fetchDomainArticles(true);
try {
fetchTopHeadlines(true); // 뉴스 초기 적재
fetchDomainArticles(true);
} catch (Exception e) {
log.error("[초기화] 데이터 초기 적재 중 오류 발생", e);
}

}

// 3시간마다 헤드라인 뉴스 Scheduling
@Scheduled(cron = "0 0 0/3 * * *")
// 4시간마다 헤드라인 뉴스 Scheduling
@Scheduled(cron = "0 0 0/4 * * *")
public void scheduledTopHeadlines() {
totalNewArticles = 0;
fetchTopHeadlines(false);
log.info("[스케줄링] 헤드라인 저장된 신규 기사: {}개", totalNewArticles);
try {
totalNewArticles = 0;
fetchTopHeadlines(false);
log.info("[스케줄링] 헤드라인 저장된 신규 기사: {}개", totalNewArticles);
} catch (Exception e) {
log.error("[스케줄링] 헤드라인 뉴스 수집 중 오류 발생", e);
}
}


@Scheduled(cron = "0 0 0 * * *")
public void scheduledFetchFixed() {
totalNewArticles = 0;
fetchDomainArticles(false);
log.info("[스케줄링] Everything 저장된 신규 기사: {}개", totalNewArticles);
try {
totalNewArticles = 0;
fetchDomainArticles(false);
log.info("[스케줄링] Everything 저장된 신규 기사: {}개", totalNewArticles);
} catch (Exception e) {
log.error("[스케줄링] Everything 뉴스 수집 중 오류 발생", e);
}
}

// 최상위 실행 메소드 ( 두 방식 동시에 테스트용 -> Scheduling 제외 )
Expand All @@ -89,14 +102,18 @@ public void fetchAndSave() {
private void fetchTopHeadlines(boolean isInit) {
for (String country : COUNTRIES) {
for (String category : CATEGORY_LIST) {
String categoryParam = (category == null) ? "" : "&category=" + category;
String apiUrl = "https://newsapi.org/v2/top-headlines?"
+ "country=" + country
+ categoryParam
+ "&pageSize=" + 100
+ "&apiKey=" + apiKey;

processApiRequest(apiUrl, country, (category == null ? "general" : category));
try {
String categoryParam = (category == null) ? "" : "&category=" + category;
String apiUrl = "https://newsapi.org/v2/top-headlines?"
+ "country=" + country
+ categoryParam
+ "&pageSize=" + 100
+ "&apiKey=" + apiKey;

processApiRequest(apiUrl, country, (category == null ? "general" : category));
} catch (Exception e) {
log.error("[헤드라인] {} / {} 처리 중 오류 발생", country, category, e);
}
}
}
if (isInit) {
Expand All @@ -106,12 +123,16 @@ private void fetchTopHeadlines(boolean isInit) {

private void fetchDomainArticles(boolean isInit) {
for (String domain : DOMAINS) {
String apiUrl = "https://newsapi.org/v2/everything?"
+ "domains=" + domain
+ "&pageSize=" + 100
+ "&apiKey=" + apiKey;
try {
String apiUrl = "https://newsapi.org/v2/everything?"
+ "domains=" + domain
+ "&pageSize=" + 100
+ "&apiKey=" + apiKey;

processApiRequest(apiUrl, "us", null);
processApiRequest(apiUrl, "us", null);
} catch (Exception e) {
log.error("[Everything] {} 도메인 처리 중 오류 발생", domain, e);
}
}
if (isInit) {
log.info("[초기 적재] Everything 저장된 신규 기사: {}개", totalNewArticles);
Expand All @@ -120,43 +141,47 @@ private void fetchDomainArticles(boolean isInit) {

// 공통 API 호출 + 저장 처리
private void processApiRequest(String apiUrl, String country, String category) {
ResponseEntity<NewsApiResponseDto> responseEntity = restTemplate.getForEntity(apiUrl, NewsApiResponseDto.class);
NewsApiResponseDto response = responseEntity.getBody();

if (response == null || response.getArticles() == null || response.getArticles().isEmpty()) {
log.info("가져올 뉴스 없음: {} / {}", (country != null ? country : "도메인"), category);
return;
}
try {
ResponseEntity<NewsApiResponseDto> responseEntity = restTemplate.getForEntity(apiUrl, NewsApiResponseDto.class);
NewsApiResponseDto response = responseEntity.getBody();

List<NewsApiArticleDto> articles = response.getArticles();
List<String> urls = articles.stream()
.map(NewsApiArticleDto::getUrl)
.collect(Collectors.toList());

// 기존 URL 조회
Set<String> existingUrls = articleRepository.findExistingUrls(urls);
if (response == null || response.getArticles() == null || response.getArticles().isEmpty()) {
log.info("가져올 뉴스 없음: {} / {}", (country != null ? country : "도메인"), category);
return;
}

// Source 캐시
List<String> sourceNames = articles.stream()
.map(NewsApiArticleDto::getSource)
.filter(Objects::nonNull)
.map(NewsApiSourceDto::getName)
.filter(Objects::nonNull)
.distinct()
.collect(Collectors.toList());

Map<String, Source> sourceCache = sourceService.preloadSources(sourceNames);

// 유효성 검증 + 저장
List<Article> articleList = articles.stream()
.filter(articleDto -> !isInvalid(articleDto) && !existingUrls.contains(articleDto.getUrl()))
.map(articleDto -> mapDtoToEntity(articleDto, country, category, sourceCache))
.collect(Collectors.toList());

if (!articleList.isEmpty()) {
articleRepository.saveAll(articleList);
log.info("{} / {} - {}개 저장 완료", (country != null ? country : "도메인"), category, articleList.size());
totalNewArticles += articleList.size();
List<NewsApiArticleDto> articles = response.getArticles();
List<String> urls = articles.stream()
.map(NewsApiArticleDto::getUrl)
.collect(Collectors.toList());

// 기존 URL 조회
Set<String> existingUrls = articleRepository.findExistingUrls(urls);

// Source 캐시
List<String> sourceNames = articles.stream()
.map(NewsApiArticleDto::getSource)
.filter(Objects::nonNull)
.map(NewsApiSourceDto::getName)
.filter(Objects::nonNull)
.distinct()
.collect(Collectors.toList());

Map<String, Source> sourceCache = sourceService.preloadSources(sourceNames);

// 유효성 검증 + 저장
List<Article> articleList = articles.stream()
.filter(articleDto -> !isInvalid(articleDto) && !existingUrls.contains(articleDto.getUrl()))
.map(articleDto -> mapDtoToEntity(articleDto, country, category, sourceCache))
.collect(Collectors.toList());

if (!articleList.isEmpty()) {
articleRepository.saveAll(articleList);
log.info("{} / {} - {}개 저장 완료", (country != null ? country : "도메인"), category, articleList.size());
totalNewArticles += articleList.size();
}
} catch (Exception e) {
log.error("[API 호출] {} / {} 처리 중 오류 발생", (country != null ? country : "도메인"), category, e);
}
}

Expand Down