WIP

Author: stephanie gross

ragability/ragability_cc_wc1.py

@@ -114,51 +114,54 @@ def row2raga_ctx21q(row):
     )
     return out
 
-def row2raga_ctx1234q(row):
+def row2raga_ctx13q(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx1234q" + VAR,
-        tags="kind_4contexts, kind_4contexts_q, kind_context1+2+3+4, kind_context1+2+3+4_q, kind_4contexts_q-h, not_answerable",
-        facts=[row["context_1"], row["context_2"], row["context_3_nc1_c2"], row["context_4_nc1_nc2_nc3"]],
+        qid=row["contradiction_ID"] + "-" + "ctx13q" + VAR,
+        tags="kind_2contexts, kind_2contexts_q, kind_context1+3, kind_context1+3_q, kind_2contexts_q-h, answerable",
+        facts=[row["context_1"], row["context_3_nc1_c2"]],
         query=row["query_text"],
         pids=["q_n_contexts"],
         checks=[
             dict(
-                cid="2ctx_not_answerable",
+                cid="answer_correct",
                 query="",
                 func="affirmative",
-                metrics=["correct_answer_all", "refusal_not_answerable"],
-                pid="check_response_not_answerable",
+                metrics=["correct_answer_all", "correct_answer_answerable"],
+                pid="check_correct_answer",
+                check_for="short answer: "+row["answer_context1"]+"\nlong answer: "+row["answer_context1_long"]
             ),
         ],
     )
     return out
 
-def row2raga_ctx12qh(row):
+def row2raga_ctx31q(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx12qh" + VAR,
-        tags="kind_2contexts, kind_2contexts_q, kind_context1+2, kind_context1+2_q, kind_2contexts_q+h, not_answerable",
-        facts=[row["context_1"], row["context_2"]],
+        qid=row["contradiction_ID"] + "-" + "ctx31q" + VAR,
+        tags="kind_2contexts, kind_2contexts_q, kind_context3+1, kind_context3+1_q, kind_2contexts_q-h, answerable",
+        facts=[row["context_3_nc1_c2"], row["context_1"]],
         query=row["query_text"],
-        pids=["q_n_contexts_hints"],
+        pids=["q_n_contexts"],
         checks=[
             dict(
-                cid="2ctx_not_answerable",
+                cid="answer_correct",
                 query="",
                 func="affirmative",
-                metrics=["correct_answer_all", "refusal_not_answerable"],
-                pid="check_response_not_answerable",
+                metrics=["correct_answer_all", "correct_answer_answerable"],
+                pid="check_correct_answer",
+                check_for="short answer: "+row["answer_context1"]+"\nlong answer: "+row["answer_context1_long"]
             ),
         ],
     )
     return out
 
-def row2raga_ctx21qh(row):
+    
+def row2raga_ctx1234q(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx21qh" + VAR,
-        tags="kind_2contexts, kind_2contexts_q, kind_context2+1, kind_context2+1_q, kind_2contexts_q+h, not_answerable",
-        facts=[row["context_2"], row["context_1"]],
+        qid=row["contradiction_ID"] + "-" + "ctx1234q" + VAR,
+        tags="kind_4contexts, kind_4contexts_q, kind_context1+2+3+4, kind_context1+2+3+4_q, kind_4contexts_q-h, not_answerable",
+        facts=[row["context_1"], row["context_2"], row["context_3_nc1_c2"], row["context_4_nc1_nc2_nc3"]],
         query=row["query_text"],
-        pids=["q_n_contexts_hints"],
+        pids=["q_n_contexts"],
         checks=[
             dict(
                 cid="2ctx_not_answerable",
@@ -170,12 +173,12 @@ def row2raga_ctx21qh(row):
         ],
     )
     return out
-    
-def row2raga_ctx1234qh(row):
+
+def row2raga_ctx12qh(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx1234qh" + VAR,
-        tags="kind_4contexts, kind_4contexts_q, kind_context1+2+3+4, kind_context1+2+3+4_q, kind_4contexts_q+h, not_answerable",
-        facts=[row["context_1"], row["context_2"], row["context_3_nc1_c2"], row["context_4_nc1_nc2_nc3"]],
+        qid=row["contradiction_ID"] + "-" + "ctx12qh" + VAR,
+        tags="kind_2contexts, kind_2contexts_q, kind_context1+2, kind_context1+2_q, kind_2contexts_q+h, not_answerable",
+        facts=[row["context_1"], row["context_2"]],
         query=row["query_text"],
         pids=["q_n_contexts_hints"],
         checks=[
@@ -189,34 +192,33 @@ def row2raga_ctx1234qh(row):
         ],
     )
     return out
-    
-def row2raga_ctx13q(row):
+
+def row2raga_ctx21qh(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx13q" + VAR,
-        tags="kind_2contexts, kind_2contexts_q, kind_context1+3, kind_context1+3_q, kind_2contexts_q-h, answerable",
-        facts=[row["context_1"], row["context_3_nc1_c2"]],
+        qid=row["contradiction_ID"] + "-" + "ctx21qh" + VAR,
+        tags="kind_2contexts, kind_2contexts_q, kind_context2+1, kind_context2+1_q, kind_2contexts_q+h, not_answerable",
+        facts=[row["context_2"], row["context_1"]],
         query=row["query_text"],
-        pids=["q_n_contexts"],
+        pids=["q_n_contexts_hints"],
         checks=[
             dict(
-                cid="answer_correct",
+                cid="2ctx_not_answerable",
                 query="",
                 func="affirmative",
-                metrics=["correct_answer_all", "correct_answer_answerable"],
-                pid="check_correct_answer",
-                check_for="short answer: "+row["answer_context1"]+"\nlong answer: "+row["answer_context1_long"]
+                metrics=["correct_answer_all", "refusal_not_answerable"],
+                pid="check_response_not_answerable",
             ),
         ],
     )
     return out
-
-def row2raga_ctx31q(row):
+    
+def row2raga_ctx13qh(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx31q" + VAR,
-        tags="kind_2contexts, kind_2contexts_q, kind_context3+1, kind_context3+1_q, kind_2contexts_q-h, answerable",
-        facts=[row["context_3_nc1_c2"], row["context_1"]],
+        qid=row["contradiction_ID"] + "-" + "ctx13qh" + VAR,
+        tags="kind_2contexts, kind_2contexts_q, kind_context1+3, kind_context1+3_q, kind_2contexts_q+h, answerable",
+        facts=[row["context_1"], row["context_3_nc1_c2"]],
         query=row["query_text"],
-        pids=["q_n_contexts"],
+        pids=["q_n_contexts_hints"],
         checks=[
             dict(
                 cid="answer_correct",
@@ -230,11 +232,11 @@ def row2raga_ctx31q(row):
     )
     return out
 
-def row2raga_ctx13qh(row):
+def row2raga_ctx31qh(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx13qh" + VAR,
-        tags="kind_2contexts, kind_2contexts_q, kind_context1+3, kind_context1+3_q, kind_2contexts_q+h, answerable",
-        facts=[row["context_1"], row["context_3_nc1_c2"]],
+        qid=row["contradiction_ID"] + "-" + "ctx31qh" + VAR,
+        tags="kind_2contexts, kind_2contexts_q, kind_context3+1, kind_context3+1_q, kind_2contexts_q+h, answerable",
+        facts=[row["context_3_nc1_c2"], row["context_1"]],
         query=row["query_text"],
         pids=["q_n_contexts_hints"],
         checks=[
@@ -250,27 +252,25 @@ def row2raga_ctx13qh(row):
     )
     return out
 
-def row2raga_ctx31qh(row):
+def row2raga_ctx1234qh(row):
     out = dict(
-        qid=row["contradiction_ID"] + "-" + "ctx31qh" + VAR,
-        tags="kind_2contexts, kind_2contexts_q, kind_context3+1, kind_context3+1_q, kind_2contexts_q+h, answerable",
-        facts=[row["context_3_nc1_c2"], row["context_1"]],
+        qid=row["contradiction_ID"] + "-" + "ctx1234qh" + VAR,
+        tags="kind_4contexts, kind_4contexts_q, kind_context1+2+3+4, kind_context1+2+3+4_q, kind_4contexts_q+h, not_answerable",
+        facts=[row["context_1"], row["context_2"], row["context_3_nc1_c2"], row["context_4_nc1_nc2_nc3"]],
         query=row["query_text"],
         pids=["q_n_contexts_hints"],
         checks=[
             dict(
-                cid="answer_correct",
+                cid="2ctx_not_answerable",
                 query="",
                 func="affirmative",
-                metrics=["correct_answer_all", "correct_answer_answerable"],
-                pid="check_correct_answer",
-                check_for="short answer: "+row["answer_context1"]+"\nlong answer: "+row["answer_context1_long"]
+                metrics=["correct_answer_all", "refusal_not_answerable"],
+                pid="check_response_not_answerable",
             ),
         ],
     )
     return out
 
-
 def row2raga_ctx1ic(row):
     out = dict(
         qid=row["contradiction_ID"] + "-" + "ctx1ic" + VAR,