Upload metrics.py with huggingface_hub

metrics.py

@@ -8,6 +8,7 @@ import evaluate
 import nltk
 import numpy
 
+from .dataclass import InternalField
 from .operator import (
  MultiStreamOperator,
  SingleStreamOperator,
@@ -60,7 +61,13 @@ class GlobalMetric(SingleStreamOperator, Metric):
 
  refs, pred = instance["references"], instance["prediction"]
 
- instance_score = self._compute([refs], [pred])
+ try:
+ instance_score = self._compute([refs], [pred])
+ except:
+ instance_score = {"score": None}
+ if isinstance(self.main_score, str) and self.main_score is not None:
+ instance_score[self.main_score] = None
+
  instance["score"]["instance"].update(instance_score)
 
  references.append(refs)
@@ -355,8 +362,27 @@ class Bleu(HuggingfaceMetric):
  scale = 1.0
 
 
+class MatthewsCorrelation(HuggingfaceMetric):
+ metric_name = "matthews_correlation"
+ main_score = "matthews_correlation"
+ str_to_id: dict = InternalField(default_factory=dict)
+
+ def get_str_id(self, str):
+ if str not in self.str_to_id:
+ id = len(self.str_to_id)
+ self.str_to_id[str] = id
+ return self.str_to_id[str]
+
+ def compute(self, references: List[List[str]], predictions: List[str]) -> dict:
+ formatted_references = [self.get_str_id(reference[0]) for reference in references]
+ formatted_predictions = [self.get_str_id(prediction) for prediction in predictions]
+ result = self.metric.compute(predictions=formatted_predictions, references=formatted_references)
+ return result
+
+
 class CustomF1(GlobalMetric):
  main_score = "f1_micro"
+ classes = None
 
  @abstractmethod
  def get_element_group(self, element):
@@ -391,6 +417,10 @@ class CustomF1(GlobalMetric):
  assert len(references) == len(predictions), (
  f"references size ({len(references)})" f" doesn't mach predictions sise ({len(references)})."
  )
+ if self.classes is None:
+ classes = set([self.get_element_group(e) for sublist in references for e in sublist])
+ else:
+ classes = self.classes
  groups_statistics = dict()
  for references_batch, predictions_batch in zip(references, predictions):
  grouped_references = self.group_elements(references_batch)
@@ -418,6 +448,7 @@ class CustomF1(GlobalMetric):
  groups_statistics[group]["recall_denominator"] += rd
 
  result = {}
+ num_of_unknown_class_predictions = 0
  pn_total = pd_total = rn_total = rd_total = 0
  for group in groups_statistics.keys():
  pn, pd, rn, rd = (
@@ -426,13 +457,21 @@ class CustomF1(GlobalMetric):
  groups_statistics[group]["recall_numerator"],
  groups_statistics[group]["recall_denominator"],
  )
- result[f"f1_{group}"] = self.f1(pn, pd, rn, rd)
  pn_total, pd_total, rn_total, rd_total = pn_total + pn, pd_total + pd, rn_total + rn, rd_total + rd
+ if group in classes:
+ result[f"f1_{group}"] = self.f1(pn, pd, rn, rd)
+ else:
+ num_of_unknown_class_predictions += pd
  try:
  result["f1_macro"] = sum(result.values()) / len(result.keys())
  except ZeroDivisionError:
  result["f1_macro"] = 1.0
 
+ amount_of_predictions = pd_total
+ if amount_of_predictions == 0:
+ result["in_classes_support"] = 1.0
+ else:
+ result["in_classes_support"] = 1.0 - num_of_unknown_class_predictions / amount_of_predictions
  result[f"f1_micro"] = self.f1(pn_total, pd_total, rn_total, rd_total)
  return result