feat: optimize label when data is empty in rag metrics (#330)

* feat: optimize label when data is empty in rag metrics

* 📚 Auto-update metrics documentation

---------

Co-authored-by: GitHub Action <[email protected]>

Author: e06084

dingo/model/llm/rag/llm_rag_answer_relevancy.py

@@ -90,8 +90,7 @@ def build_messages(cls, input_data: Data) -> List:
         raw_data = getattr(input_data, 'raw_data', {})
         answer = input_data.content or raw_data.get("answer", "")
 
-        if not answer:
-            raise ValueError("Answer Relevancy评估需要answer字段")
+        # 注意: answer 为空的情况已在 eval() 方法中处理，这里假设 answer 非空
 
         # 使用json.dumps()来安全转义响应字符串
         import json
@@ -223,6 +222,19 @@ def calculate_score(cls, answers: List[Dict[str, Any]], original_question: str)
     def eval(cls, input_data: Data) -> EvalDetail:
         """评估答案相关性"""
         raw_data = getattr(input_data, 'raw_data', {})
+
+        # 检查 answer 是否为空
+        answer = input_data.content or raw_data.get("answer", "")
+        if not answer:
+            # 如果 answer 为空，直接返回 0 分
+            log.warning("Answer Relevancy 评估: answer 字段为空，直接返回 0 分")
+            result = EvalDetail(metric=cls.__name__)
+            result.score = 0.0
+            result.status = True
+            result.label = ["QUALITY_BAD.ANSWER_RELEVANCY_NO_ANSWER"]
+            result.reason = ["answer 字段为空，无法评估答案相关性，分数设为 0"]
+            return result
+
         # 提取原始问题
         original_question = input_data.prompt or raw_data.get("question", "")
         if not original_question:

dingo/model/llm/rag/llm_rag_context_precision.py

@@ -166,8 +166,7 @@ def build_messages(cls, input_data: Data) -> List:
         question = input_data.prompt or raw_data.get("question", "")
         answer = input_data.content or raw_data.get("answer", "")
 
-        if not answer:
-            raise ValueError("Context Precision评估需要answer字段")
+        # 注意: answer 为空的情况已在 eval() 方法中处理，这里假设 answer 非空
 
         # 处理contexts
         contexts = None
@@ -277,6 +276,20 @@ def eval(cls, input_data: Data) -> EvalDetail:
         if cls.client is None:
             cls.create_client()
 
+        # 检查 answer 是否为空
+        raw_data = getattr(input_data, 'raw_data', {})
+        answer = input_data.content or raw_data.get("answer", "")
+
+        if not answer:
+            # 如果 answer 为空，直接返回 0 分
+            log.warning("Context Precision 评估: answer 字段为空，直接返回 0 分")
+            result = EvalDetail(metric=cls.__name__)
+            result.score = 0.0
+            result.status = True
+            result.label = ["QUALITY_BAD.CONTEXT_PRECISION_NO_ANSWER"]
+            result.reason = ["answer 字段为空，无法评估上下文精度，分数设为 0"]
+            return result
+
         # 获取所有上下文的消息
         messages_list = cls.build_messages(input_data)
         responses = []

dingo/model/llm/rag/llm_rag_context_recall.py

@@ -145,11 +145,11 @@ def build_messages(cls, input_data: Data) -> List:
             else:
                 contexts = [raw_contexts]
 
-        if not expected_output:
-            raise ValueError("Context Recall评估需要expected_output或answer字段")
         if not contexts:
             raise ValueError("Context Recall评估需要contexts字段")
 
+        # 注意: expected_output 为空的情况已在 eval() 方法中处理，这里假设 expected_output 非空
+
         # 拼接上下文
         combined_contexts = "\n\n".join([f"上下文{i + 1}:\n{ctx}" for i, ctx in enumerate(contexts)])
 
@@ -228,3 +228,29 @@ def process_response(cls, response: str) -> EvalDetail:
             result.reason = [f"上下文召回评估未通过 (分数: {score:.2f}/10)\n{reason_text}"]
 
         return result
+
+    @classmethod
+    def eval(cls, input_data: Data) -> EvalDetail:
+        """重写父类的eval方法，添加对expected_output的检查"""
+        if cls.client is None:
+            cls.create_client()
+
+        # 检查 expected_output 或 answer 是否为空
+        raw_data = getattr(input_data, 'raw_data', {})
+        expected_output = raw_data.get("expected_output", "")
+        if not expected_output:
+            # 如果没有 expected_output，尝试使用 content 或 answer
+            expected_output = input_data.content or raw_data.get("answer", "")
+
+        if not expected_output:
+            # 如果 expected_output 和 answer 都为空，直接返回 0 分
+            log.warning("Context Recall 评估: expected_output 和 answer 字段均为空，直接返回 0 分")
+            result = EvalDetail(metric=cls.__name__)
+            result.score = 0.0
+            result.status = True
+            result.label = ["QUALITY_BAD.CONTEXT_RECALL_NO_REFERENCE"]
+            result.reason = ["expected_output 和 answer 字段均为空，无法评估上下文召回率，分数设为 0"]
+            return result
+
+        # 调用父类的 eval 方法
+        return super().eval(input_data)

docs/metrics.md

@@ -118,6 +118,12 @@ This document provides comprehensive information about all quality metrics used
 | `RESUME_QUALITY_BAD_PROFESSIONALISM` | RuleResumeEmoji, RuleResumeInformal | Detects emoji usage in resume which reduces professionalism; Detects informal or colloquial expressions in resume | Internal Implementation | N/A | N/A |
 | `RESUME_QUALITY_BAD_STRUCTURE` | RuleResumeNameMissing, RuleResumeSectionMissing | Checks if resume contains a name in the first 200 characters; Checks if resume contains required sections like educat... | Internal Implementation | N/A | N/A |
 
+### SFT Data Assessment Metrics - Agent-Enhanced
+
+| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
+|------|--------|-------------|--------------|-------------------|----------|
+| `AgentHallucination` | AgentHallucination | Agent-based hallucination detection with automatic web search for missing context | Internal Implementation | N/A | N/A |
+
 ### Text Generation
 
 | Type | Metric | Description | Paper Source | Evaluation Results | Examples |

examples/rag/dataset_rag_eval_baseline.py

@@ -39,7 +39,7 @@
 OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
 OPENAI_URL = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
 OPENAI_KEY = os.getenv("OPENAI_API_KEY", "")
-EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "text-embedding-3-large")
+EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "doubao-embedding-large-text-250515")
 
 # 数据文件路径
 INPUT_DATA_PATH = str(PROJECT_ROOT / "test/data/fiqa.jsonl")  # 或 "test/data/ragflow_eval_data_50.jsonl"