Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@
import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordResponse;
import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordSource;
import com.jobdri.jobdri_api.domain.analysis.type.QuestionAnalysisStatus;
import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient;
import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient.AnalysisAiCallResult;
import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisPromptInput;
import com.jobdri.jobdri_api.domain.analysis.service.ai.JobCategoryEvaluationCriteriaProvider;
import com.jobdri.jobdri_api.domain.analysis.service.core.AnalysisResultConstants;
import com.jobdri.jobdri_api.domain.analysis.service.sanitization.AnalysisSanitizationRules;
import lombok.RequiredArgsConstructor;
import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand;
import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult;
import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator;
import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

Expand All @@ -31,7 +30,6 @@
import java.util.Set;

@Service
@RequiredArgsConstructor
@Slf4j
public class EvaluationAnalysisBatchService {
private static final Long EVALUATION_QUESTION_ID = 1L;
Expand All @@ -46,9 +44,31 @@ public class EvaluationAnalysisBatchService {
"answer"
);

private final AnalysisAiClient analysisAiClient;
private final JobCategoryEvaluationCriteriaProvider jobCategoryEvaluationCriteriaProvider;
private final EvaluationAnalysisGenerator evaluationAnalysisGenerator;
private final ObjectMapper objectMapper;
private final EvaluationSanitizationService evaluationSanitizationService;

@Autowired
public EvaluationAnalysisBatchService(
EvaluationAnalysisGenerator evaluationAnalysisGenerator,
ObjectMapper objectMapper,
EvaluationSanitizationService evaluationSanitizationService
) {
this.evaluationAnalysisGenerator = evaluationAnalysisGenerator;
this.objectMapper = objectMapper;
this.evaluationSanitizationService = evaluationSanitizationService;
}

EvaluationAnalysisBatchService(
EvaluationAnalysisGenerator evaluationAnalysisGenerator,
ObjectMapper objectMapper
) {
this(
evaluationAnalysisGenerator,
objectMapper,
new EvaluationSanitizationService()
);
}

public EvaluationBatchSummary run(Path inputPath, Path outputPath) throws IOException {
List<EvaluationAnalysisCase> cases = readCases(inputPath);
Expand Down Expand Up @@ -106,27 +126,17 @@ private void validateHeaders(List<String> headers) {
}

private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCase) {
AnalysisPromptInput promptInput = new AnalysisPromptInput(
EvaluationGeneratedResult generatedResult = evaluationAnalysisGenerator.generate(new EvaluationAnalysisCommand(
evaluationCase.caseId(),
"평가용 회사",
evaluationCase.jobCategoryMiddle(),
evaluationCase.jobCategorySmall(),
evaluationCase.mainTasks(),
evaluationCase.qualifications(),
evaluationCase.preferences(),
List.of(new AnalysisPromptInput.QuestionAnswer(
EVALUATION_QUESTION_ID,
evaluationCase.question(),
evaluationCase.answer()
))
);

AnalysisAiCallResult aiCallResult = analysisAiClient.analyzeForEvaluationResult(
promptInput,
jobCategoryEvaluationCriteriaProvider
.findByMiddleName(evaluationCase.jobCategoryMiddle())
.orElse(null)
);
AnalysisLlmResponse llmResponse = aiCallResult.response();
evaluationCase.question(),
evaluationCase.answer()
));
AnalysisLlmResponse llmResponse = generatedResult.response();

int jobFit = validateScore("jobFit", llmResponse == null ? null : llmResponse.jobFit());
int impact = validateScore("impact", llmResponse == null ? null : llmResponse.impact());
Expand All @@ -136,7 +146,7 @@ private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCa
log.debug(
"Evaluation serialized missing keyword flow. caseId={}, candidateMissingKeywordCount={}, finalMissingKeywordCount={}, evaluationSerializedMissingKeywordCount={}",
evaluationCase.caseId(),
size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().missingKeywordCandidates()),
size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()),
size(llmResponse == null ? null : llmResponse.missingKeywords()),
missingKeywords.size()
);
Expand All @@ -158,25 +168,25 @@ private EvaluationAnalysisResult analyzeCase(EvaluationAnalysisCase evaluationCa
writeJson(missingKeywords),
writeJson(questionAnalyses),
writeJson(llmResponse),
writeJson(aiCallResult.rawCandidateResponse()),
writeJson(aiCallResult.sanitizedCandidateResponse()),
writeJson(aiCallResult.candidateReviewResponse()),
size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().analysisCandidates()),
size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().analysisCandidates()),
size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().strengthCandidates()),
size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().missingKeywordCandidates()),
acceptedDecisionCount(aiCallResult.candidateReviewResponse()),
rejectedDecisionCount(aiCallResult.candidateReviewResponse()),
rejectionCodeCounts(aiCallResult.candidateReviewResponse()),
writeJson(generatedResult.rawCandidateResponse()),
writeJson(generatedResult.sanitizedCandidateResponse()),
writeJson(generatedResult.candidateReviewResponse()),
size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().analysisCandidates()),
size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().analysisCandidates()),
size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().strengthCandidates()),
size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()),
acceptedDecisionCount(generatedResult.candidateReviewResponse()),
rejectedDecisionCount(generatedResult.candidateReviewResponse()),
rejectionCodeCounts(generatedResult.candidateReviewResponse()),
questionAnalyses.size(),
size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().strengthCandidates()),
size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().strengthCandidates()),
llmResponse.keyStrengths() == null ? 0 : llmResponse.keyStrengths().size(),
size(aiCallResult.sanitizedCandidateResponse() == null ? null : aiCallResult.sanitizedCandidateResponse().missingKeywordCandidates()),
size(generatedResult.sanitizedCandidateResponse() == null ? null : generatedResult.sanitizedCandidateResponse().missingKeywordCandidates()),
missingKeywords.size(),
aiCallResult.candidateCallLatencyMs(),
aiCallResult.finalCallLatencyMs(),
aiCallResult.candidateCallLatencyMs(),
aiCallResult.finalCallLatencyMs(),
generatedResult.candidateCallLatencyMs(),
generatedResult.finalCallLatencyMs(),
generatedResult.candidateCallLatencyMs(),
generatedResult.finalCallLatencyMs(),
null,
null,
null,
Expand Down Expand Up @@ -214,15 +224,15 @@ private List<MissingKeywordResponse> buildMissingKeywords(
if (source.isEmpty()) {
continue;
}
if (!AnalysisSanitizationRules.isValidMissingKeyword(
if (!evaluationSanitizationService.isValidMissingKeyword(
keyword,
source.get(),
evaluationCase.mainTasks(),
evaluationCase.qualifications()
)) {
continue;
}
if (AnalysisSanitizationRules.isMissingKeywordMentionedInAnswers(
if (evaluationSanitizationService.isMissingKeywordMentionedInAnswers(
keyword,
evaluationCase.answer()
)) {
Expand Down Expand Up @@ -275,11 +285,11 @@ private List<EvaluationQuestionAnalysisResult> buildQuestionAnalyses(
continue;
}
if (status == QuestionAnalysisStatus.PROVEN
&& !AnalysisSanitizationRules.hasValidProvenReason(item.reason())) {
&& !evaluationSanitizationService.hasValidProvenReason(item.reason())) {
continue;
}
if (status == QuestionAnalysisStatus.FABRICATED
&& !AnalysisSanitizationRules.hasFabricatedDirectConflictEvidence(
&& !evaluationSanitizationService.hasFabricatedDirectConflictEvidence(
item.sentence(),
item.reason()
)) {
Expand Down Expand Up @@ -435,7 +445,7 @@ private String normalizeImprovement(
String improvement,
QuestionAnalysisStatus status
) {
return AnalysisSanitizationRules.normalizeImprovement(
return evaluationSanitizationService.normalizeImprovement(
sentence,
answer,
improvement,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
import com.jobdri.jobdri_api.domain.analysis.dto.external.llm.AnalysisCandidateResponse;
import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationDecision;
import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizationResult;
import com.jobdri.jobdri_api.domain.analysis.service.sanitization.MissingKeywordSanitizer;
import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

Expand All @@ -23,7 +24,6 @@
import java.util.stream.Collectors;

@Service
@RequiredArgsConstructor
@Slf4j
class MissingKeywordSanitizerReplayService {
private static final List<String> REQUIRED_HEADERS = List.of(
Expand Down Expand Up @@ -76,6 +76,20 @@ class MissingKeywordSanitizerReplayService {
);

private final ObjectMapper objectMapper;
private final EvaluationSanitizationService evaluationSanitizationService;

@Autowired
MissingKeywordSanitizerReplayService(
ObjectMapper objectMapper,
EvaluationSanitizationService evaluationSanitizationService
) {
this.objectMapper = objectMapper;
this.evaluationSanitizationService = evaluationSanitizationService;
}

MissingKeywordSanitizerReplayService(ObjectMapper objectMapper) {
this(objectMapper, new EvaluationSanitizationService());
}

ReplaySummary replay(Path input, Path output, Path reviewOutput) throws IOException {
List<String> headers = EvaluationCsvSupport.readHeaders(input);
Expand Down Expand Up @@ -112,7 +126,7 @@ ReplaySummary replay(Path input, Path output, Path reviewOutput) throws IOExcept
);
List<AnalysisCandidateResponse.MissingKeywordCandidate> rawCandidates =
safeMissingKeywordCandidates(rawResponse);
MissingKeywordSanitizationResult replayResult = MissingKeywordSanitizer.sanitize(
MissingKeywordSanitizationResult replayResult = evaluationSanitizationService.sanitizeMissingKeywordCandidates(
value(row, "mainTasks"),
value(row, "qualifications"),
value(row, "answer"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import com.jobdri.jobdri_api.domain.evaluation.analysis.mapper.EvaluationMissingKeywordSourceMapper;
import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeyword;
import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationMissingKeywordSource;
import com.jobdri.jobdri_api.domain.analysis.service.sanitization.AnalysisSanitizationRules;
import com.jobdri.jobdri_api.domain.evaluation.analysis.sanitization.EvaluationSanitizationService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.NestedExceptionUtils;
Expand Down Expand Up @@ -66,15 +66,22 @@ class NlgEvaluationBatchService {
private final NlgEvaluationAiClient nlgEvaluationAiClient;
private final ObjectMapper objectMapper;
private final EvaluationLlmSnapshotParser evaluationLlmSnapshotParser;
private final EvaluationSanitizationService evaluationSanitizationService;

@Autowired
NlgEvaluationBatchService(
NlgEvaluationAiClient nlgEvaluationAiClient,
ObjectMapper objectMapper
ObjectMapper objectMapper,
EvaluationSanitizationService evaluationSanitizationService
) {
this.nlgEvaluationAiClient = nlgEvaluationAiClient;
this.objectMapper = objectMapper;
this.evaluationLlmSnapshotParser = new EvaluationLlmSnapshotParser(objectMapper);
this.evaluationSanitizationService = evaluationSanitizationService;
}

NlgEvaluationBatchService(NlgEvaluationAiClient nlgEvaluationAiClient, ObjectMapper objectMapper) {
this(nlgEvaluationAiClient, objectMapper, new EvaluationSanitizationService());
}

NlgEvaluationSummary run(Path inputPath, Path outputPath) throws IOException {
Expand Down Expand Up @@ -446,8 +453,8 @@ private Optional<MissingKeywordMissInvalidReason> missingKeywordMissInvalidReaso
if (source.isEmpty() || source.get() == EvaluationMissingKeywordSource.PREFERENCE) {
return Optional.of(MissingKeywordMissInvalidReason.INVALID_SOURCE);
}
if (AnalysisSanitizationRules.isStructuredQualificationKeyword(evaluation.keyword())
|| AnalysisSanitizationRules.isStructuredQualificationKeyword(evaluation.relatedRequirement())) {
if (evaluationSanitizationService.isStructuredQualificationKeyword(evaluation.keyword())
|| evaluationSanitizationService.isStructuredQualificationKeyword(evaluation.relatedRequirement())) {
return Optional.of(MissingKeywordMissInvalidReason.STRUCTURED_QUALIFICATION);
}
String sourceText = source.get() == EvaluationMissingKeywordSource.MAIN_TASK
Expand All @@ -456,7 +463,7 @@ private Optional<MissingKeywordMissInvalidReason> missingKeywordMissInvalidReaso
if (!containsNormalized(sourceText, evaluation.relatedRequirement())) {
return Optional.of(MissingKeywordMissInvalidReason.RELATED_REQUIREMENT_NOT_IN_JD);
}
if (!AnalysisSanitizationRules.isValidMissingKeyword(
if (!evaluationSanitizationService.isValidMissingKeyword(
evaluation.keyword(),
EvaluationMissingKeywordSourceMapper.toAnalysisSource(source.get()),
input.mainTasks(),
Expand Down Expand Up @@ -862,7 +869,7 @@ private boolean containsNormalized(String sourceText, String fragment) {
}

private String normalize(String value) {
return AnalysisSanitizationRules.normalizeText(value);
return evaluationSanitizationService.normalizeText(value);
}

private String truncateShortRationale(String rationale) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package com.jobdri.jobdri_api.domain.evaluation.analysis.adapter;

import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient;
import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisAiClient.AnalysisAiCallResult;
import com.jobdri.jobdri_api.domain.analysis.service.ai.AnalysisPromptInput;
import com.jobdri.jobdri_api.domain.analysis.service.ai.JobCategoryEvaluationCriteriaProvider;
import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationAnalysisCommand;
import com.jobdri.jobdri_api.domain.evaluation.analysis.model.EvaluationGeneratedResult;
import com.jobdri.jobdri_api.domain.evaluation.analysis.port.EvaluationAnalysisGenerator;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;

import java.util.List;

@Component
@RequiredArgsConstructor
public class AnalysisAiEvaluationAnalysisGenerator implements EvaluationAnalysisGenerator {
private static final Long EVALUATION_QUESTION_ID = 1L;

private final AnalysisAiClient analysisAiClient;
private final JobCategoryEvaluationCriteriaProvider jobCategoryEvaluationCriteriaProvider;

@Override
public EvaluationGeneratedResult generate(EvaluationAnalysisCommand command) {
AnalysisPromptInput promptInput = new AnalysisPromptInput(
command.caseId(),
"평가용 회사",
command.jobCategorySmall(),
command.mainTasks(),
command.qualifications(),
command.preferences(),
List.of(new AnalysisPromptInput.QuestionAnswer(
EVALUATION_QUESTION_ID,
command.question(),
command.answer()
))
);

AnalysisAiCallResult aiCallResult = analysisAiClient.analyzeForEvaluationResult(
promptInput,
jobCategoryEvaluationCriteriaProvider
.findByMiddleName(command.jobCategoryMiddle())
.orElse(null)
);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
return new EvaluationGeneratedResult(
aiCallResult.response(),
aiCallResult.rawCandidateResponse(),
aiCallResult.sanitizedCandidateResponse(),
aiCallResult.candidateReviewResponse(),
aiCallResult.candidateCallLatencyMs(),
aiCallResult.finalCallLatencyMs()
);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package com.jobdri.jobdri_api.domain.evaluation.analysis.model;

public record EvaluationAnalysisCommand(
String caseId,
String jobCategoryMiddle,
String jobCategorySmall,
String mainTasks,
String qualifications,
String preferences,
String question,
String answer
) {
}
Loading
Loading