Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -63,4 +63,6 @@ interface ArticleRepository : ListCrudRepository<Article, Long> {
@Param("nextId") nextId: Long?,
@Param("limit") limit: Int,
): List<ArticleWithBoard>

fun existsByOriginLink(originLink: String): Boolean
}
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ abstract class BaseCrawler(
val rawLink = getPostLink(row)
val detailUrl = if (rawLink.startsWith("http")) rawLink else "$baseUrl$rawLink"

if (crawlerRepository.existsByOriginLink(detailUrl)) {
if (articleRepository.existsByOriginLink(detailUrl)) {
continue
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package com.wafflestudio.team2server.crawler.controller

import com.wafflestudio.team2server.crawler.BaseCrawler
import com.wafflestudio.team2server.crawler.dto.CrawlerStatusResponse
import com.wafflestudio.team2server.crawler.service.CrawlerService
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.PathVariable
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestMapping
Expand All @@ -10,6 +13,7 @@ import org.springframework.web.bind.annotation.RestController
@RestController
@RequestMapping("/api/crawlers")
class CrawlerController(
private val crawlerService: CrawlerService,
private val crawlers: List<BaseCrawler>,
) {
@PostMapping("/{crawlerCode}/run")
Expand All @@ -27,4 +31,7 @@ class CrawlerController(
return ResponseEntity.internalServerError().body(" 실행 실패: ${e.message}")
}
}

@GetMapping
fun getCrawlerStatus(): CrawlerStatusResponse = crawlerService.getAllCrawlerStatus()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package com.wafflestudio.team2server.crawler.dto

import java.time.Instant

data class CrawlerStatusResponse(
val count: Int,
val results: List<CrawlerInfo>,
) {
data class CrawlerInfo(
val id: Long,
val boardName: String,
val lastUpdatedAt: Instant,
val nextUpdateAt: Instant,
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,5 @@ data class Crawler(
val nextUpdateAt: Instant,
val createdAt: Instant? = null,
val updatedAt: Instant? = null,
val code: String,
)
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
package com.wafflestudio.team2server.crawler.repository

import com.wafflestudio.team2server.article.model.Article
import com.wafflestudio.team2server.crawler.model.Crawler
import org.springframework.data.jdbc.repository.query.Modifying
import org.springframework.data.jdbc.repository.query.Query
import org.springframework.data.repository.ListCrudRepository
import org.springframework.data.repository.query.Param
import java.time.Instant

interface CrawlerRepository : ListCrudRepository<Article, Long> {
fun existsByOriginLink(originLink: String): Boolean

interface CrawlerRepository : ListCrudRepository<Crawler, Long> {
@Modifying
@Query("UPDATE crawlers SET updated_at = :now, next_update_at = :next WHERE board_id = :boardId")
fun updateLastCrawledAt(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package com.wafflestudio.team2server.crawler.service

import com.wafflestudio.team2server.crawler.dto.CrawlerStatusResponse
import com.wafflestudio.team2server.crawler.repository.CrawlerRepository
import org.springframework.stereotype.Service
import java.time.Instant

@Service
class CrawlerService(
private val crawlerRepository: CrawlerRepository,
) {
fun getAllCrawlerStatus(): CrawlerStatusResponse {
val crawlers = crawlerRepository.findAll()

val crawlerInfos =
crawlers.map { crawler ->
CrawlerStatusResponse.CrawlerInfo(
id = crawler.id ?: 0L,
boardName = crawler.code,
lastUpdatedAt = crawler.updatedAt ?: Instant.now(),
nextUpdateAt = crawler.nextUpdateAt,
)
}

return CrawlerStatusResponse(
count = crawlerInfos.size,
results = crawlerInfos,
)
}
}
17 changes: 17 additions & 0 deletions src/test/kotlin/com/wafflestudio/team2server/CrawlerTests.kt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.wafflestudio.team2server

import com.wafflestudio.team2server.crawler.repository.CrawlerRepository
import com.wafflestudio.team2server.crawler.service.MysnuCrawlerService
import org.junit.jupiter.api.Test
import org.mockito.BDDMockito.given
Expand All @@ -10,7 +11,10 @@ import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc
import org.springframework.test.context.ActiveProfiles
import org.springframework.test.context.bean.override.mockito.MockitoBean
import org.springframework.test.web.servlet.MockMvc
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post
import org.springframework.test.web.servlet.result.MockMvcResultHandlers.print
import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath
import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
import org.testcontainers.junit.jupiter.Testcontainers

Expand All @@ -22,6 +26,7 @@ class CrawlerTests
@Autowired
constructor(
private val mvc: MockMvc,
private val crawlerRepository: CrawlerRepository,
) {
@MockitoBean
private lateinit var mysnuCrawlerService: MysnuCrawlerService
Expand All @@ -37,4 +42,16 @@ class CrawlerTests

verify(mysnuCrawlerService).crawl()
}

@Test
fun `get crawler status returns ok and correct body structure`() {
mvc
.perform(
get("/api/crawlers"),
).andDo(print())
.andExpect(status().isOk)
.andExpect(jsonPath("$.count").isNumber)
.andExpect(jsonPath("$.results").isArray)
.andExpect(jsonPath("$.results[0].boardName").exists())
}
}