Wangcx

evals/completion_fns/uni_finder.py

@@ -93,7 +93,7 @@ def __call__(self, prompt: Union[str, list[dict]], **kwargs: Any) -> UniFinderCo
  "query": prompt,
  'api_key': self.api_key
  }
- response = requests.post(url, json=payload, timeout=1200)
+ response = requests.post(url, json=payload, timeout=300)
  try:
  answer = response.json()['answer']
  except:

evals/elsuite/rag_table_extract.py

@@ -91,97 +91,109 @@ def __init__(
  self.instructions = instructions
 
  def eval_sample(self, sample, rng):
- assert isinstance(sample, FileSample)
-
- prompt = \
- self.instructions
- # + f"\nThe fields should at least contain {sample.compare_fields}"
- result = self.completion_fn(
- prompt=prompt,
- temperature=0.0,
- max_tokens=5,
- file_name=sample.file_name,
- file_link=sample.file_link
- )
- sampled = result.get_completions()[0]
-
- compare_fields_types = [type(x) for x in sample.compare_fields]
- header_rows = [0, 1] if tuple in compare_fields_types else [0]
-
- correct_answer = parse_table_multiindex(pd.read_csv(sample.answerfile_name, header=header_rows).astype(str), compare_fields=sample.compare_fields)
- correct_answer.to_csv("temp.csv", index=False)
- correct_str = open("temp.csv", 'r').read()
-
- if sample.index not in correct_answer.columns:
- if len(header_rows)>1:
- correct_answer.columns = pd.MultiIndex.from_tuples([sample.index] + list(correct_answer.columns)[1:])
- else:
- correct_answer.columns = [sample.index] + list(correct_answer.columns)[1:]
-
  try:
- if re.search(outlink_pattern, sampled) is not None:
- code = re.search(outlink_pattern, sampled).group()
- link = re.sub(outlink_pattern, r"\1", code)
-
- fname = f"/tmp/LLMEvals_{uuid.uuid4()}.csv"
- os.system(f"wget {link} -O {fname}")
- table = pd.read_csv(fname)
- if pd.isna(table.iloc[0, 0]):
- table = pd.read_csv(fname, header=header_rows)
- elif "csv" in prompt:
- code = re.search(csv_pattern, sampled).group()
- code_content = re.sub(csv_pattern, r"\1", code)
- code_content_processed = parse_csv_text(code_content)
- # table = pd.read_csv(StringIO(code_content_processed), header=header_rows)
- table = pd.read_csv(StringIO(code_content_processed))
- if pd.isna(table.iloc[0, 0]):
- table = pd.read_csv(StringIO(code_content_processed), header=header_rows)
-
- elif "json" in prompt:
- code = re.search(json_pattern, sampled).group()
- code_content = re.sub(json_pattern, r"\1", code).replace("\"", "")
- table = pd.DataFrame(json.loads(code_content))
- else:
- table = pd.DataFrame()
-
- table = parse_table_multiindex(table, compare_fields=sample.compare_fields)
 
- if sample.index not in table.columns:
+ assert isinstance(sample, FileSample)
+
+ prompt = \
+ self.instructions
+ # + f"\nThe fields should at least contain {sample.compare_fields}"
+ result = self.completion_fn(
+ prompt=prompt,
+ temperature=0.0,
+ max_tokens=5,
+ file_name=sample.file_name,
+ file_link=sample.file_link
+ )
+ sampled = result.get_completions()[0]
+
+ compare_fields_types = [type(x) for x in sample.compare_fields]
+ header_rows = [0, 1] if tuple in compare_fields_types else [0]
+
+ correct_answer = parse_table_multiindex(pd.read_csv(sample.answerfile_name, header=header_rows).astype(str), compare_fields=sample.compare_fields)
+ correct_answer.to_csv("temp.csv", index=False)
+ correct_str = open("temp.csv", 'r').read()
+
+ if sample.index not in correct_answer.columns:
  if len(header_rows)>1:
- table.columns = pd.MultiIndex.from_tuples([sample.index] + list(table.columns)[1:])
+ correct_answer.columns = pd.MultiIndex.from_tuples([sample.index] + list(correct_answer.columns)[1:])
  else:
- table.columns =[sample.index] + list(table.columns)[1:]
-
- print(table)
- answerfile_out = sample.answerfile_name.replace(".csv", "_output.csv")
- table.to_csv(answerfile_out, index=False)
- picked_str = open(answerfile_out, 'r').read()
- except:
- print(Path(sample.file_name).stem)
- traceback.print_exc()
+ correct_answer.columns = [sample.index] + list(correct_answer.columns)[1:]
+
+ try:
+ if re.search(outlink_pattern, sampled) is not None:
+ code = re.search(outlink_pattern, sampled).group()
+ link = re.sub(outlink_pattern, r"\1", code)
+
+ fname = f"/tmp/LLMEvals_{uuid.uuid4()}.csv"
+ os.system(f"wget {link} -O {fname}")
+ table = pd.read_csv(fname)
+ if pd.isna(table.iloc[0, 0]):
+ table = pd.read_csv(fname, header=header_rows)
+ elif "```csv" in prompt:
+ code = re.search(csv_pattern, sampled).group()
+ code_content = re.sub(csv_pattern, r"\1", code)
+ code_content_processed = parse_csv_text(code_content)
+ # table = pd.read_csv(StringIO(code_content_processed), header=header_rows)
+ table = pd.read_csv(StringIO(code_content_processed))
+ if pd.isna(table.iloc[0, 0]):
+ table = pd.read_csv(StringIO(code_content_processed), header=header_rows)
+
+ elif "```json" in prompt:
+ code = re.search(json_pattern, sampled).group()
+ code_content = re.sub(json_pattern, r"\1", code).replace("\"", "")
+ table = pd.DataFrame(json.loads(code_content))
+ else:
+ table = pd.DataFrame()
+
+ table = parse_table_multiindex(table, compare_fields=sample.compare_fields)
+
+ if sample.index not in table.columns:
+ if len(header_rows)>1:
+ table.columns = pd.MultiIndex.from_tuples([sample.index] + list(table.columns)[1:])
+ else:
+ table.columns =[sample.index] + list(table.columns)[1:]
+
+ print(table)
+ print(correct_answer)
+ answerfile_out = sample.answerfile_name.replace(".csv", "_output.csv")
+ table.to_csv(answerfile_out, index=False)
+ picked_str = open(answerfile_out, 'r').read()
+ except:
+ print(Path(sample.file_name).stem)
+ traceback.print_exc()
+ record_match(
+ prompt=prompt,
+ correct=False,
+ expected=correct_str,
+ picked=sampled,
+ file_name=sample.file_name,
+ jobtype="match_all"
+ )
+ table = None
+ picked_str = "Failed to parse"
+
+ metrics = tableMatching(correct_answer, table, index=sample.index, compare_fields=sample.compare_fields,
+ record=False, file_name=sample.file_name)
  record_match(
  prompt=prompt,
- correct=False,
+ correct=(metrics["recall_field"] == 1.0 and metrics["recall_index"] == 1.0 and metrics["recall_value"] == 1.0),
  expected=correct_str,
- picked=sampled,
+ picked=picked_str,
  file_name=sample.file_name,
  jobtype="match_all"
  )
+ return metrics
+ except:
+ print(Path(sample.file_name).stem)
+ traceback.print_exc()
+
  table = None
  picked_str = "Failed to parse"
 
- metrics = tableMatching(correct_answer, table, index=sample.index, compare_fields=sample.compare_fields,
- record=False, file_name=sample.file_name)
- record_match(
- prompt=prompt,
- correct=(metrics["recall_field"] == 1.0 and metrics["recall_index"] == 1.0 and metrics["recall_value"] == 1.0),
- expected=correct_str,
- picked=picked_str,
- file_name=sample.file_name,
- jobtype="match_all"
- )
- return metrics
- return metrics
+ metrics = {"recall_field": 0.0, "recall_index": 0.0, "recall_value": 0.0, "recall_value_strict": 0.0,
+ "accuracy_value": 0.0, "accuracy_value_strict": 0.0, "recall_SMILES": 0.0}
+ return metrics
 
  def run(self, recorder: RecorderBase):
  raw_samples = get_rag_dataset(self._prefix_registry_path(self.samples_jsonl).as_posix())

evals/elsuite/utils.py

@@ -236,11 +236,11 @@ def is_float(str):
  return True
  elif fuzzy_normalize_value(a) == fuzzy_normalize_value(b):
  return True
- # elif ((a[-2:] in unit_str or a[-1] in unit_str or a.split()[-1] in unit_str) and
- #  (b[-2:] in unit_str or b[-1] in unit_str or b.split()[-1] in unit_str)):
- #  a = standardize_unit(a)
- #  b = standardize_unit(b)
- #  return a == b
+ elif ((a[-2:] in unit_str or a[-1] in unit_str or a.split()[-1] in unit_str) and
+ (b[-2:] in unit_str or b[-1] in unit_str or b.split()[-1] in unit_str)):
+ a = standardize_unit(a)
+ b = standardize_unit(b)
+ return a == b
  elif a.lower() in nan_str and b.lower() in nan_str:
  return True
  if ((a.lower().startswith(b.lower()) or a.lower().endswith(b.lower())) or
@@ -271,9 +271,9 @@ def fuzzy_normalize_name(s):
  s = re.sub(r'[^\w\s.\-\(\)]', '', s)
  if s in synonyms:
  s = synonyms[s]
-  
- if "+" in s:
- s = s.replace("+","")
+
+ if "°" in s:
+ s = s.replace("°","")
 
  # 分割字符串为单词列表
  words = s.split()
@@ -348,7 +348,7 @@ def match_indices(ind0, ind1, threshold=0.9) -> dict:
  Match the indices of two dataframes.
  """
  renames = {}
- name2query = lambda name: name if type(name) != tuple else name[0] if name[1] == "" else name[1]
+ name2query = lambda name: name if type(name) != tuple else name[0] if len(name)==1 or name[1] == "" else name[1]
  similarities = np.array(np.ones([len(ind0) + 15, len(ind1) + 15]), dtype=np.float64)
  querys0 = [name2query(name) for name in ind0]
  querys1 = [name2query(name) for name in ind1]
@@ -423,12 +423,11 @@ def match_indices(ind0, ind1, threshold=0.9) -> dict:
  for idx in df_ref.index:
  _total_matching = 1.0
  for col in compare_fields_:
- gt = df_ref.loc[idx, col]
- gt = str(gt[0]) if type(gt) == pd.Series else str(gt)
  try:
- p = df_prompt.loc[idx, col]
- p = str(p[0]) if type(p) == pd.Series else str(p)
+ gt = str(df_ref.loc[idx, col])
+ p = str(df_prompt.loc[idx, col])
  except:
+ gt = 'error'
  p = 'not found'
 
  _is_matching = fuzzy_compare_name(gt, p, compare_value=True) if col != "SMILES" else compare_molecule(gt, p)