Skip to content

헤드리스 경로의 LLM 후보 홉 실패 로그 추가 #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
github-actions wants to merge 1 commit into
base: main
Choose a base branch
from
Open

헤드리스 경로의 LLM 후보 홉 실패 로그 추가 #129

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 36 additions & 4 deletions src/main/java/com/depromeet/piki/extractor/extraction/HeadlessProductLinkExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.depromeet.piki.extractor.domain.ProductLink;
import com.depromeet.piki.extractor.domain.ProductSnapshot;
import com.depromeet.piki.extractor.domain.ProductSnapshotException;
import com.depromeet.piki.extractor.extraction.headless.HeadlessRenderException;
import com.depromeet.piki.extractor.extraction.headless.HeadlessRenderer;
import com.depromeet.piki.extractor.extraction.headless.RenderedHop;
Expand Down Expand Up @@ -40,7 +41,7 @@ public class HeadlessProductLinkExtractor implements LinkExtractionStrategy {
private record Candidate(RenderedHop hop, String html) {
}

private record Chosen(PageContent page, StructuredExtraction result) {
private record Chosen(Candidate candidate, PageContent page, StructuredExtraction result) {
}

@Override
Expand Down Expand Up @@ -72,15 +73,15 @@ public ProductSnapshot extract(ProductLink link, boolean authorized, String mode
continue;
}
if (first == null) {
first = new Chosen(page, result);
first = new Chosen(candidate, page, result);
}
if (fallback == null && !LlmInputGate.hasNothingForLlm(page.document())) {
fallback = new Chosen(page, result);
fallback = new Chosen(candidate, page, result);
}
}
Chosen chosen = fallback != null ? fallback : first;
if (chosen != null) {
return htmlSnapshotPipeline.extract(chosen.page(), chosen.result(), timing, model);
return extractChosen(chosen, link, hops, timing, model);
}
if (blocked) {
log.warn("headless hops unusable blocked=true hops={} url={}", hops.size(), link.safeLogString());
Expand All @@ -94,6 +95,37 @@ public ProductSnapshot extract(ProductLink link, boolean authorized, String mode
throw HeadlessRenderException.upstream("렌더 HTML 이 없다", null);
}

/**
* 고른 홉이 파이프라인에서 실패하면 어느 홉이 LLM 에 갔는지 남긴다. 성공 쪽은 extract 원장이 지지만 실패 쪽은
* 요청 URL 만 남아, 차단 페이지가 신호를 피해 후보로 남았는지와 상품 홉을 두고 다른 홉을 골랐는지를 가를 수
* 없었다. title 은 페이지 본문이라 남기지 않고 경로·status·분량으로만 가른다.
*/
private ProductSnapshot extractChosen(
Chosen chosen,
ProductLink link,
List<RenderedHop> hops,
String timing,
String model
) {
try {
return htmlSnapshotPipeline.extract(chosen.page(), chosen.result(), timing, model);
} catch (ProductSnapshotException e) {
RenderedHop hop = chosen.candidate().hop();
log.info(
"headless chosen hop failed code={} hop={}/{} status={} source={} html={}chars hopUrl={} url={}",
e.code(),
hops.indexOf(hop) + 1,
hops.size(),
hop.status(),
chosen.candidate().html().equals(hop.dom()) ? "dom" : "body",
chosen.page().retainedChars(),
chosen.page().finalUrl().safeLogString(),
link.safeLogString()
);
throw e;
}
}

private static List<Candidate> candidates(List<RenderedHop> hops) {
List<Candidate> candidates = new ArrayList<>();
for (RenderedHop hop : hops.reversed()) {
Expand Down
Loading