fix: Selenium Headless 브라우저 사용 #26

Author: juny0955

build.gradle

@@ -31,6 +31,7 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-validation'
     implementation 'com.github.ben-manes.caffeine:caffeine:3.2.0'
     implementation 'org.apache.httpcomponents.client5:httpclient5:5.4.3'
+    implementation 'org.seleniumhq.selenium:selenium-java:4.31.0'
     implementation 'org.jsoup:jsoup:1.18.3'
     implementation 'com.rometools:rome:2.1.0'
     implementation 'org.springframework.boot:spring-boot-starter-security'

src/main/java/com/davcatch/devcatch/admin/service/schduler/SchedulerService.java

@@ -1,5 +1,6 @@
 package com.davcatch.devcatch.admin.service.schduler;
 
+import lombok.Builder;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -30,22 +31,24 @@ public String runArticleCreationScheduler() {
 			articleSchedulerService.createNewArticle();
 
 			// 성공 이력 저장
-			SchedulerExecution execution = new SchedulerExecution();
-			execution.setSchedulerName("아티클 수집 및 생성");
-			execution.setExecutionTime(startTime);
-			execution.setStatus("성공");
-			execution.setDetails("수동 실행으로 신규 아티클 수집 완료");
+			SchedulerExecution execution = SchedulerExecution.builder()
+				.schedulerName("아티클 수집 및 생성")
+				.executionTime(startTime)
+				.status("성공")
+				.details("수동 실행으로 신규 아티클 수집 완료")
+				.build();
 
 			return "아티클 수집 및 생성 스케줄러가 성공적으로 실행되었습니다.";
 		} catch (Exception e) {
 			log.error("아티클 생성 스케줄러 실행 중 오류: {}", e.getMessage(), e);
 
 			// 실패 이력 저장
-			SchedulerExecution execution = new SchedulerExecution();
-			execution.setSchedulerName("아티클 수집 및 생성");
-			execution.setExecutionTime(startTime);
-			execution.setStatus("실패");
-			execution.setDetails("오류: " + e.getMessage());
+			SchedulerExecution execution = SchedulerExecution.builder()
+				.schedulerName("아티클 수집 및 생성")
+				.executionTime(startTime)
+				.status("실패")
+				.details("오류: " + e.getMessage())
+				.build();
 
 			return "아티클 수집 중 오류가 발생했습니다: " + e.getMessage();
 		}
@@ -63,22 +66,24 @@ public String runArticleNotificationScheduler() {
 			articleNotificationService.sendNewArticle();
 
 			// 성공 이력 저장
-			SchedulerExecution execution = new SchedulerExecution();
-			execution.setSchedulerName("아티클 이메일 발송");
-			execution.setExecutionTime(startTime);
-			execution.setStatus("성공");
-			execution.setDetails("수동 실행으로 이메일 발송 완료");
+			SchedulerExecution execution = SchedulerExecution.builder()
+				.schedulerName("아티클 이메일 발송")
+				.executionTime(startTime)
+				.status("성공")
+				.details("수동 실행으로 이메일 발송 완료")
+				.build();
 
 			return "아티클 이메일 발송 스케줄러가 성공적으로 실행되었습니다.";
 		} catch (Exception e) {
 			log.error("아티클 발송 스케줄러 실행 중 오류: {}", e.getMessage(), e);
 
 			// 실패 이력 저장
-			SchedulerExecution execution = new SchedulerExecution();
-			execution.setSchedulerName("아티클 이메일 발송");
-			execution.setExecutionTime(startTime);
-			execution.setStatus("실패");
-			execution.setDetails("오류: " + e.getMessage());
+			SchedulerExecution execution = SchedulerExecution.builder()
+				.schedulerName("아티클 이메일 발송")
+				.executionTime(startTime)
+				.status("실패")
+				.details("오류: " + e.getMessage())
+				.build();
 
 			return "아티클 이메일 발송 중 오류가 발생했습니다: " + e.getMessage();
 		}
@@ -88,6 +93,7 @@ public String runArticleNotificationScheduler() {
 	 * 스케줄러 실행 이력 DTO
 	 */
 	@Data
+	@Builder
 	public static class SchedulerExecution {
 		private String schedulerName;
 		private LocalDateTime executionTime;

src/main/java/com/davcatch/devcatch/common/config/GPTConfig.java

@@ -31,16 +31,19 @@ public Optional<Document> getDocument(String link) {
 			ResponseEntity<String> response = crawlingRestTemplate.getForEntity(link, String.class);
 			String html = response.getBody();
 
-			if (html != null) {
-				Document document = Jsoup.parse(html);
-				log.debug("크롤링 정상 수집 : {}", link);
-
-				return Optional.of(document);
+			if (html == null || html.isBlank()) {
+				log.warn("[{}] 해당 페이지를 가져올 수 없습니다", link);
+				return Optional.empty();
 			}
+
+			Document document = Jsoup.parse(html);
+			log.debug("크롤링 정상 수집 : {}", link);
+
+			return Optional.of(document);
+
 		} catch (Exception e) {
 			log.error("({}) 크롤링 중 에러 발생 : {}", link, e.getMessage());
+			return Optional.empty();
 		}
-
-		return Optional.empty();
 	}
 }

src/main/java/com/davcatch/devcatch/common/integration/crawling/WebCrawler.java

@@ -17,7 +17,7 @@
 @Slf4j
 public class GptSummaryService {
 
-	private static final String URL = "https://api.openai.com/v1/chat/completions";
+	private static final String GPT_API_URL = "https://api.openai.com/v1/chat/completions";
 
 	private final RestTemplate gptApiRestTemplate;
 
@@ -35,11 +35,13 @@ public class GptSummaryService {
 	public GptResponse getSummary(String content) throws CustomException {
 		log.debug("GPT API 요청 시작");
 
-		GptResponse response;
+		GptResponse response = null;
 
 		try {
 			GptRequest request = GptRequest.create(content, model, sysPrompt);
-			response = gptApiRestTemplate.postForObject(URL, request, GptResponse.class);
+			System.out.println("gptApiRestTemplate.postForObject(GPT_API_URL, request, GptResponse.class) = "
+				+ gptApiRestTemplate.postForObject(GPT_API_URL, request, GptResponse.class));
+			response = gptApiRestTemplate.postForObject(GPT_API_URL, request, GptResponse.class);
 		} catch (Exception e) {
 			log.error("GPT API 요청중 에러 발생 : {}", e.getMessage());
 			throw new CustomException(ErrorCode.GPT_REQUEST_ERROR);

src/main/java/com/davcatch/devcatch/common/integration/gpt/GptSummaryService.java

@@ -2,9 +2,11 @@
 
 import java.util.List;
 
+import lombok.AllArgsConstructor;
 import lombok.Data;
 
 @Data
+@AllArgsConstructor
 public class GptResponse {
 
 	private List<Choices> choices;

src/main/java/com/davcatch/devcatch/common/integration/gpt/response/GptResponse.java

@@ -21,41 +21,37 @@
 public class RssReaderService {
 
 	private final RestTemplate rssRestTemplate;
-	private final RestTemplate cloudflareRssRestTemplate;
 
 	/**
 	 * RSS FEED 파싱
-	 * @param feedUrl Feed URL
+	 * @param source 해당 소스
 	 * @return Feed
 	 */
 	public Optional<SyndFeed> reader(Source source) {
-		log.debug("RSS FEED 수집 시작 : {}", source.getFeedUrl());
+		log.debug("[{}] RSS FEED 수집 시작", source.getName());
 
 		try {
-			ResponseEntity<String> response;
-			if (source.getName().equals("우아한형제들")) {
-				cloudflareRssRestTemplate.getForEntity(source.getMainUrl(), String.class);
-				response = cloudflareRssRestTemplate.getForEntity(source.getFeedUrl(), String.class);
-			} else
-				response = rssRestTemplate.getForEntity(source.getFeedUrl(), String.class);
-
-			String xml = response.getBody();
-
-			if (xml != null) {
-				SyndFeedInput syndFeedInput = new SyndFeedInput();
-				syndFeedInput.setAllowDoctypes(true);
-				syndFeedInput.setPreserveWireFeed(true);
-				syndFeedInput.setXmlHealerOn(true); // XML 문법 오류 자동 복구 활성화
-
-				SyndFeed feed = syndFeedInput.build(new StringReader(xml));
-				log.debug("RSS FEED 정상 수집 : {}", source.getName());
-				return Optional.of(feed);
+			ResponseEntity<String> response = rssRestTemplate.getForEntity(source.getFeedUrl(), String.class);
+
+			String rssFeedXml = response.getBody();
+
+			if (rssFeedXml == null || rssFeedXml.isBlank()) {
+				log.warn("[{}] RSS FEED를 가져올 수 없습니다", source.getName());
+				return Optional.empty();
 			}
 
+			SyndFeedInput syndFeedInput = new SyndFeedInput();
+			syndFeedInput.setAllowDoctypes(true);
+			syndFeedInput.setPreserveWireFeed(true);
+			syndFeedInput.setXmlHealerOn(true); // XML 문법 오류 자동 복구 활성화
+
+			SyndFeed feed = syndFeedInput.build(new StringReader(rssFeedXml));
+			log.debug("RSS FEED 정상 수집 : {}", source.getName());
+			return Optional.of(feed);
+
 		} catch (FeedException e) {
 			log.error("[{}] RSS FEED 수집중 오류 발생 : {}", source.getName(), e.getMessage());
+			return Optional.empty();
 		}
-
-		return Optional.empty();
 	}
 }

src/main/java/com/davcatch/devcatch/common/integration/rss/RssReaderService.java

@@ -0,0 +1,105 @@
+package com.davcatch.devcatch.common.integration.selenium;
+
+import java.io.StringReader;
+import java.util.Optional;
+
+import org.jsoup.Jsoup;
+import org.openqa.selenium.JavascriptExecutor;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+import org.openqa.selenium.support.ui.WebDriverWait;
+import org.springframework.stereotype.Service;
+
+import com.davcatch.devcatch.common.exception.CustomException;
+import com.davcatch.devcatch.common.exception.ErrorCode;
+import com.davcatch.devcatch.domain.source.Source;
+import com.rometools.rome.feed.synd.SyndFeed;
+import com.rometools.rome.io.SyndFeedInput;
+
+import jakarta.annotation.PostConstruct;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+public class SeleniumBrowserService {
+
+	private WebDriver createWebDriver() throws CustomException {
+		try {
+			ChromeOptions options = new ChromeOptions();
+			options.addArguments("--headless=new");
+			options.addArguments("--disable-gpu");
+			options.addArguments("--no-sandbox");
+			options.addArguments("--disable-dev-shm-usage");
+
+			options.addArguments("--disable-blink-features=AutomationControlled");
+			options.setExperimentalOption("excludeSwitches", new String[]{"enable-automation"});
+			options.setExperimentalOption("useAutomationExtension", false);
+
+			WebDriver webDriver = new ChromeDriver(options);
+
+			// 자동화 감지 방지 스크립트
+			((JavascriptExecutor) webDriver).executeScript("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})");
+
+			log.info("셀레니움 헤드리스 브라우저 초기화 성공");
+			return webDriver;
+		} catch (Exception e) {
+			log.error("셀레니움 헤드리스 브라우저 초기화 실패: {}", e.getMessage());
+			throw new CustomException(ErrorCode.SERVER_ERROR);
+		}
+	}
+
+	public void destroyWebDriver(WebDriver webDriver) {
+		if (webDriver != null) {
+			try {
+				webDriver.quit();
+				log.info("셀레니움 헤드리스 브라우저 정상 종료");
+			} catch (Exception e) {
+				log.error("셀레니움 헤드리스 브라우저 종료 중 오류: {}", e.getMessage());
+			}
+		}
+	}
+
+	public Optional<SyndFeed> reader(Source source) {
+
+		log.debug("[{}] 셀레니움 헤드리스 RSS FEED 수집 시작", source.getName());
+
+		WebDriver webDriver = null;
+		try {
+			webDriver = createWebDriver();
+
+			// 페이지 접근
+			webDriver.get(source.getFeedUrl());
+
+			// 페이지 로딩 대기
+			WebDriverWait wait = new WebDriverWait(webDriver, java.time.Duration.ofSeconds(10));
+			wait.until(d -> ((JavascriptExecutor)d).executeScript("return document.readyState").equals("complete"));
+
+			// Cloudflare 우회 대기
+			Thread.sleep(3000);
+
+			// 페이지 소스 가져오기
+			String pageSource = webDriver.getPageSource();
+
+			if (pageSource == null || pageSource.isBlank()) {
+				log.warn("[{}] 페이지 소스를 가져올 수 없습니다", source.getName());
+				return Optional.empty();
+			}
+
+			// XML 파싱
+			SyndFeedInput syndFeedInput = new SyndFeedInput();
+			syndFeedInput.setAllowDoctypes(true);
+			syndFeedInput.setPreserveWireFeed(true);
+			syndFeedInput.setXmlHealerOn(true); // XML 문법 오류 자동 복구 활성화
+
+			String parse = Jsoup.parse(pageSource).body().text();
+			SyndFeed feed = syndFeedInput.build(new StringReader(parse));
+			return Optional.of(feed);
+		} catch (Exception e) {
+			log.error("[{}] RSS FEED 수집중 오류 발생 : {}", source.getName(), e.getMessage());
+			return Optional.empty();
+		} finally {
+			destroyWebDriver(webDriver);
+		}
+	}
+}

src/main/java/com/davcatch/devcatch/common/integration/selenium/SeleniumBrowserService.java

@@ -6,6 +6,7 @@
 
 import org.springframework.stereotype.Service;
 
+import com.davcatch.devcatch.common.integration.selenium.SeleniumBrowserService;
 import com.davcatch.devcatch.domain.source.Source;
 import com.davcatch.devcatch.web.service.source.SourceService;
 
@@ -20,24 +21,26 @@ public class ArticleSchedulerService {
 	private final SourceService sourceService;
 	private final ArticleSchedulerTask articleSchedulerTask;
 	private final Executor schedulerTaskExecutor;
+	private final SeleniumBrowserService seleniumBrowserService;
 
 	public void createNewArticle() {
 		List<Source> sources = sourceService.getActiveSources();
 		log.info("총 {}개 소스 처리 시작", sources.size());
 
 		int batchSize = 5;
 
-		for (int i=0; i<sources.size(); i+=batchSize) {
+		for (int i = 0; i < sources.size(); i += batchSize) {
 			int endIndex = Math.min(i + batchSize, sources.size());
 			List<Source> batchSources = sources.subList(i, endIndex);
 
 			List<CompletableFuture<Void>> futures = batchSources.stream()
 				.map(source -> CompletableFuture.runAsync(() -> {
-						articleSchedulerTask.processSource(source);
+					articleSchedulerTask.processSource(source);
 				}, schedulerTaskExecutor))
 				.toList();
 
 			CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
 		}
+
 	}
 }