Upload processors.py with huggingface_hub

processors.py

@@ -1,8 +1,9 @@
 import json
 import re
-from typing import Any
+from difflib import get_close_matches
+from typing import Any, Dict
 
-from .operators import FieldOperator
+from .operators import FieldOperator, InstanceFieldOperator
 
 
 class ToString(FieldOperator):
@@ -117,6 +118,28 @@ class Capitalize(FieldOperator):
  return text.capitalize()
 
 
+class GetStringAfter(FieldOperator):
+ substring: str
+
+ def process_value(self, text: Any) -> Any:
+ return text.split(self.substring, 1)[-1].strip()
+
+
+class MatchClosestOption(InstanceFieldOperator):
+ options_field: str = "options"
+
+ def process_instance_value(self, value: Any, instance: Dict[str, Any]):
+ options = instance["task_data"][self.options_field]
+ return get_close_matches(value, options, n=1, cutoff=0.0)[0]
+
+
+def process_instance_value(self, value, instance):
+ options = instance[self.options_field]
+ # Get the closest match; n=1 returns the single closest match
+ closest_match = get_close_matches(value, options, n=1, cutoff=0)
+ return closest_match[0] if closest_match else None
+
+
 class Substring(FieldOperator):
  begin: int = 0
  end: int = None
@@ -152,6 +175,13 @@ class YesNoToInt(FieldOperator):
  return text
 
 
+class YesToOneElseZero(FieldOperator):
+ def process_value(self, text: Any) -> Any:
+ if text == "yes":
+ return "1"
+ return "0"
+
+
 class StrToFloatFormat(FieldOperator):
  def process_value(self, text: Any) -> Any:
  try: