Merge pull request #26 from scuuy/main

add 3 types of running of reasoning pipelines in statics, has been tested using `dataflow init` folder

Author: SunnyHaze

dataflow/statics/pipelines/api_pipelines/reasoning_pipeline.py

@@ -53,7 +53,7 @@ def __init__(self):
             llm_serving=llm_serving
         )
         ########################## branch ############################
-        self.answer_pipeline_root_step6 = AnswerPipelineRoot()
+        # self.answer_pipeline_root_step6 = AnswerPipelineRoot() 
         ########################## answer ############################
         self.answer_generator_step7 = AnswerGenerator(
             llm_serving=llm_serving
@@ -103,11 +103,11 @@ def forward(self):
             output_key = "question_category"
         )
         ############# branch #############
-        self.answer_pipeline_root_step6.run(
-            storage = self.storage.step(),
-            input_answer_key = "output",
-            input_gt_key = "golden_answer"
-        )
+        # self.answer_pipeline_root_step6.run(
+        #     storage = self.storage.step(),
+        #     input_answer_key = "output",
+        #     input_gt_key = "golden_answer"
+        # )
         ############## answer #############
         self.answer_generator_step7.run(
             storage = self.storage.step(),

dataflow/statics/pipelines/cpu_pipelines/reasoning_pipeline.py

@@ -1,4 +1,8 @@
-from dataflow.operators.process.Reasoning import AnswerFormatterFilter
+from dataflow.operators.process.Reasoning import (
+    AnswerFormatterFilter,
+    AnswerGroundTruthFilter,
+    AnswerNgramFilter,
+)
 from dataflow.utils.storage import FileStorage
 
 class ReasoningPipeline():
@@ -13,10 +17,30 @@ def __init__(self):
 
         self.answer_format_filter_step1 = AnswerFormatterFilter()
 
+        self.answer_groundtruth_filter_step2 = AnswerGroundTruthFilter()
+        
+        self.answer_ngram_filter_step3 = AnswerNgramFilter(
+            min_score = 0.1,
+            max_score = 1.0,
+            ngrams = 5
+        )
+        
     def forward(self):
         self.answer_format_filter_step1.run(
             storage = self.storage.step(),
-            input_key = "generated_cot",
+            input_key = "output",
+        )
+        
+        self.answer_groundtruth_filter_step2.run(
+            storage = self.storage.step(),
+            test_answer_key = "output",
+            gt_answer_key =  "golden_answer"
+        )
+        
+        self.answer_ngram_filter_step3.run(
+            storage = self.storage.step(),
+            question_key = "instruction",
+            answer_key = "output"
         )
 
 if __name__ == "__main__":

dataflow/statics/pipelines/gpu_pipelines/reasoning_pipeline.py

@@ -27,8 +27,8 @@ def __init__(self):
         )
 
         llm_serving = LocalModelLLMServing(
-            model_name_or_path="Qwen2.5-7B-Instruct", # set to your own model path
-            tensor_parallel_size=4,
+            model_name_or_path="Qwen/Qwen2.5-7B-Instruct", # set to your own model path
+            tensor_parallel_size=1,
             max_tokens=8192,
             model_source="local"
         )
@@ -52,7 +52,7 @@ def __init__(self):
             llm_serving=llm_serving
         )
         ########################## branch ############################
-        self.answer_pipeline_root_step6 = AnswerPipelineRoot()
+        # self.answer_pipeline_root_step6 = AnswerPipelineRoot()
         ########################## answer ############################
         self.answer_generator_step7 = AnswerGenerator(
             llm_serving=llm_serving
@@ -102,11 +102,11 @@ def forward(self):
             output_key = "question_category"
         )
         ############# branch #############
-        self.answer_pipeline_root_step6.run(
-            storage = self.storage.step(),
-            input_answer_key = "output",
-            input_gt_key = "golden_answer"
-        )
+        # self.answer_pipeline_root_step6.run(
+        #     storage = self.storage.step(),
+        #     input_answer_key = "output",
+        #     input_gt_key = "golden_answer"
+        # )
         ############## answer #############
         self.answer_generator_step7.run(
             storage = self.storage.step(),