From 4350ee623f880c6ca6ce9c3f975dde466681f6ae Mon Sep 17 00:00:00 2001 From: YsK-dev Date: Sat, 14 Nov 2026 14:08:46 +0300 Subject: [PATCH 1/4] Measure search quality on the BEIR SciFact benchmark evaluation/retrieval_eval.py stores each SciFact abstract as one passage in the app's own engine and runs the 300 test questions through vector search, BM25, vector + reranker, the full hybrid + reranker pipeline, and the app's default call (3 passages, confidence cut-off 0.3). Datasets and models are pinned. Results in evaluation/results.json, per question in evaluation/per_question_scifact.csv. Metric checks in tests/test_retrieval_metrics.py. --- evaluation/metrics.py | 36 + evaluation/per_question_scifact.csv | 1801 +++++++++++++++++++++++++++ evaluation/results.json | 84 ++ evaluation/retrieval_eval.py | 209 ++++ tests/test_retrieval_metrics.py | 59 + 5 files changed, 2189 insertions(+) create mode 100644 evaluation/metrics.py create mode 100644 evaluation/per_question_scifact.csv create mode 100644 evaluation/results.json create mode 100644 evaluation/retrieval_eval.py create mode 100644 tests/test_retrieval_metrics.py diff --git a/evaluation/metrics.py b/evaluation/metrics.py new file mode 100644 index 0000000..f13b391 --- /dev/null +++ b/evaluation/metrics.py @@ -0,0 +1,36 @@ +"""Ranking metrics for one question. Plain Python, no dependencies. + +`ranked` is the list of document ids the system returned, best first. +`relevant` maps each relevant document id to its relevance grade (1 or more). +""" + +import math + + +def ndcg_at_k(ranked: list[str], relevant: dict[str, int], k: int = 10) -> float: + """Normalised discounted cumulative gain at k, as in the BEIR benchmark.""" + dcg = sum(relevant.get(doc, 0) / math.log2(rank + 2) for rank, doc in enumerate(ranked[:k])) + ideal = sorted(relevant.values(), reverse=True)[:k] + idcg = sum(grade / math.log2(rank + 2) for rank, grade in enumerate(ideal)) + return dcg / idcg if idcg > 0 else 0.0 + + +def recall_at_k(ranked: list[str], relevant: dict[str, int], k: int = 10) -> float: + """Share of the relevant documents that appear in the top k.""" + wanted = {doc for doc, grade in relevant.items() if grade > 0} + if not wanted: + return 0.0 + return len(wanted & set(ranked[:k])) / len(wanted) + + +def hit_at_k(ranked: list[str], relevant: dict[str, int], k: int = 3) -> float: + """1.0 if at least one relevant document is in the top k, else 0.0.""" + return float(any(relevant.get(doc, 0) > 0 for doc in ranked[:k])) + + +def mrr_at_k(ranked: list[str], relevant: dict[str, int], k: int = 10) -> float: + """1 / rank of the first relevant document in the top k, or 0.0 if there is none.""" + for rank, doc in enumerate(ranked[:k]): + if relevant.get(doc, 0) > 0: + return 1.0 / (rank + 1) + return 0.0 diff --git a/evaluation/per_question_scifact.csv b/evaluation/per_question_scifact.csv new file mode 100644 index 0000000..4a45ab1 --- /dev/null +++ b/evaluation/per_question_scifact.csv @@ -0,0 +1,1801 @@ +setup,query_id,returned,ms,ndcg10,recall10,mrr10,hit3 +dense,1,10,9.68054,0.38685,1.0,0.2,0.0 +dense,3,10,9.9485,0.5,1.0,0.33333,1.0 +dense,5,10,9.0325,0.63093,1.0,0.5,1.0 +dense,13,10,9.73871,0.0,0.0,0.0,0.0 +dense,36,10,11.26167,0.49115,1.0,0.33333,1.0 +dense,42,10,9.91908,0.63093,1.0,0.5,1.0 +dense,48,10,8.64708,0.0,0.0,0.0,0.0 +dense,49,10,9.86017,1.0,1.0,1.0,1.0 +dense,50,10,9.69054,1.0,1.0,1.0,1.0 +dense,51,10,9.57742,1.0,1.0,1.0,1.0 +dense,53,10,9.72192,1.0,1.0,1.0,1.0 +dense,54,10,9.63629,1.0,1.0,1.0,1.0 +dense,56,10,10.07879,1.0,1.0,1.0,1.0 +dense,57,10,9.3725,1.0,1.0,1.0,1.0 +dense,70,10,9.55012,0.0,0.0,0.0,0.0 +dense,72,10,9.80296,1.0,1.0,1.0,1.0 +dense,75,10,9.55596,0.63093,1.0,0.5,1.0 +dense,94,10,8.96471,0.5,1.0,0.33333,1.0 +dense,99,10,9.29692,0.63093,1.0,0.5,1.0 +dense,100,10,9.58583,0.5,1.0,0.33333,1.0 +dense,113,10,9.50358,1.0,1.0,1.0,1.0 +dense,115,10,9.27246,1.0,1.0,1.0,1.0 +dense,118,10,11.43742,0.63093,1.0,0.5,1.0 +dense,124,10,10.30162,1.0,1.0,1.0,1.0 +dense,127,10,9.63,1.0,1.0,1.0,1.0 +dense,128,10,9.60729,0.0,0.0,0.0,0.0 +dense,129,10,8.95175,1.0,1.0,1.0,1.0 +dense,130,10,9.49037,1.0,1.0,1.0,1.0 +dense,132,10,9.54454,0.0,0.0,0.0,0.0 +dense,133,10,9.61846,0.63995,0.6,1.0,1.0 +dense,137,10,9.51329,1.0,1.0,1.0,1.0 +dense,141,10,9.42083,0.91972,1.0,1.0,1.0 +dense,142,10,9.86571,1.0,1.0,1.0,1.0 +dense,143,10,9.44371,1.0,1.0,1.0,1.0 +dense,146,10,11.00546,1.0,1.0,1.0,1.0 +dense,148,10,8.79483,0.63093,1.0,0.5,1.0 +dense,163,10,9.50354,1.0,1.0,1.0,1.0 +dense,171,10,9.84758,1.0,1.0,1.0,1.0 +dense,179,10,8.99012,0.95583,1.0,1.0,1.0 +dense,180,10,10.50846,1.0,1.0,1.0,1.0 +dense,183,10,8.53538,1.0,1.0,1.0,1.0 +dense,185,10,9.06525,1.0,1.0,1.0,1.0 +dense,198,10,9.37179,0.0,0.0,0.0,0.0 +dense,208,10,9.18637,0.5,1.0,0.33333,1.0 +dense,212,10,8.85512,0.35621,1.0,0.16667,0.0 +dense,213,10,10.33663,0.38685,1.0,0.2,0.0 +dense,216,10,9.03325,1.0,1.0,1.0,1.0 +dense,217,10,9.53658,1.0,1.0,1.0,1.0 +dense,218,10,9.21404,1.0,1.0,1.0,1.0 +dense,219,10,9.3155,1.0,1.0,1.0,1.0 +dense,230,10,9.10767,1.0,1.0,1.0,1.0 +dense,232,10,9.0945,1.0,1.0,1.0,1.0 +dense,233,10,9.04679,1.0,1.0,1.0,1.0 +dense,236,10,8.77438,1.0,1.0,1.0,1.0 +dense,237,10,9.10054,1.0,1.0,1.0,1.0 +dense,238,10,9.97,0.63093,1.0,0.5,1.0 +dense,239,10,9.04325,0.0,0.0,0.0,0.0 +dense,248,10,9.58875,0.5,1.0,0.33333,1.0 +dense,249,10,9.70213,0.5,1.0,0.33333,1.0 +dense,261,10,11.02521,0.20438,0.5,0.14286,0.0 +dense,268,10,8.12212,1.0,1.0,1.0,1.0 +dense,269,10,8.17967,1.0,1.0,1.0,1.0 +dense,274,10,10.61404,1.0,1.0,1.0,1.0 +dense,275,10,9.52712,0.67973,1.0,0.5,1.0 +dense,279,10,9.00908,1.0,1.0,1.0,1.0 +dense,294,10,9.40754,0.0,0.0,0.0,0.0 +dense,295,10,9.14608,0.5,1.0,0.33333,1.0 +dense,298,10,8.93442,1.0,1.0,1.0,1.0 +dense,300,10,9.91088,0.43068,1.0,0.25,0.0 +dense,303,10,8.95063,0.0,0.0,0.0,0.0 +dense,312,10,10.54762,0.0,0.0,0.0,0.0 +dense,314,10,10.12454,0.0,0.0,0.0,0.0 +dense,324,10,9.10613,0.63093,1.0,0.5,1.0 +dense,327,10,10.08621,1.0,1.0,1.0,1.0 +dense,338,10,10.25475,0.63093,1.0,0.5,1.0 +dense,343,10,10.12292,0.83155,1.0,1.0,1.0 +dense,350,10,8.83946,1.0,1.0,1.0,1.0 +dense,354,10,8.95508,1.0,1.0,1.0,1.0 +dense,362,10,10.29908,1.0,1.0,1.0,1.0 +dense,380,10,8.84217,1.0,1.0,1.0,1.0 +dense,384,10,8.97638,0.0,0.0,0.0,0.0 +dense,385,10,9.00012,0.69343,1.0,0.5,1.0 +dense,386,10,10.02521,1.0,1.0,1.0,1.0 +dense,388,10,8.70112,0.63093,1.0,0.5,1.0 +dense,399,10,9.97079,1.0,1.0,1.0,1.0 +dense,410,10,9.88042,1.0,1.0,1.0,1.0 +dense,411,10,8.96533,1.0,1.0,1.0,1.0 +dense,415,10,9.35758,0.0,0.0,0.0,0.0 +dense,421,10,8.98279,0.0,0.0,0.0,0.0 +dense,431,10,9.69217,0.0,0.0,0.0,0.0 +dense,436,10,8.78946,1.0,1.0,1.0,1.0 +dense,437,10,8.67092,0.0,0.0,0.0,0.0 +dense,439,10,8.853,1.0,1.0,1.0,1.0 +dense,440,10,9.07258,1.0,1.0,1.0,1.0 +dense,443,10,9.17042,1.0,1.0,1.0,1.0 +dense,452,10,10.84625,0.17724,0.5,0.1,0.0 +dense,475,10,9.43958,0.0,0.0,0.0,0.0 +dense,478,10,10.48404,1.0,1.0,1.0,1.0 +dense,491,10,8.8025,1.0,1.0,1.0,1.0 +dense,501,10,10.09521,1.0,1.0,1.0,1.0 +dense,502,10,9.10092,0.0,0.0,0.0,0.0 +dense,507,10,8.86829,0.63093,1.0,0.5,1.0 +dense,508,10,8.9325,0.0,0.0,0.0,0.0 +dense,513,10,9.94712,0.63093,1.0,0.5,1.0 +dense,514,10,9.544,1.0,1.0,1.0,1.0 +dense,516,10,10.38388,1.0,1.0,1.0,1.0 +dense,517,10,9.22208,0.0,0.0,0.0,0.0 +dense,521,10,9.91183,1.0,1.0,1.0,1.0 +dense,525,10,9.40804,1.0,1.0,1.0,1.0 +dense,527,10,10.47533,0.0,0.0,0.0,0.0 +dense,528,10,10.70675,1.0,1.0,1.0,1.0 +dense,532,10,9.34075,1.0,1.0,1.0,1.0 +dense,533,10,9.31912,1.0,1.0,1.0,1.0 +dense,535,10,9.38425,0.43068,1.0,0.25,0.0 +dense,536,10,9.14192,1.0,1.0,1.0,1.0 +dense,539,10,9.63604,1.0,1.0,1.0,1.0 +dense,540,10,10.00492,0.26407,0.5,0.25,0.0 +dense,544,10,9.15292,0.0,0.0,0.0,0.0 +dense,549,10,9.15188,1.0,1.0,1.0,1.0 +dense,551,10,9.70779,0.63093,1.0,0.5,1.0 +dense,552,10,9.8205,1.0,1.0,1.0,1.0 +dense,554,10,9.47238,0.0,0.0,0.0,0.0 +dense,560,10,9.20363,0.0,0.0,0.0,0.0 +dense,569,10,9.33037,0.43068,1.0,0.25,0.0 +dense,575,10,9.2365,0.5,1.0,0.33333,1.0 +dense,577,10,10.27333,0.31546,1.0,0.125,0.0 +dense,578,10,9.38229,1.0,1.0,1.0,1.0 +dense,587,10,10.41129,0.0,0.0,0.0,0.0 +dense,589,10,9.51662,1.0,1.0,1.0,1.0 +dense,593,10,9.69929,1.0,1.0,1.0,1.0 +dense,597,10,9.92012,0.79287,1.0,1.0,1.0 +dense,598,10,9.49717,0.33333,1.0,0.14286,0.0 +dense,613,10,9.19129,1.0,1.0,1.0,1.0 +dense,619,10,9.20262,0.0,0.0,0.0,0.0 +dense,623,10,9.97762,0.0,0.0,0.0,0.0 +dense,628,10,9.45025,0.43068,1.0,0.25,0.0 +dense,636,10,10.62654,1.0,1.0,1.0,1.0 +dense,637,10,8.84067,0.43068,1.0,0.25,0.0 +dense,641,10,9.35792,0.83155,1.0,1.0,1.0 +dense,644,10,9.0545,0.0,0.0,0.0,0.0 +dense,649,10,9.27513,1.0,1.0,1.0,1.0 +dense,659,10,10.28037,0.43068,1.0,0.25,0.0 +dense,660,10,9.48775,0.0,0.0,0.0,0.0 +dense,674,10,9.29629,0.33333,1.0,0.14286,0.0 +dense,684,10,8.78875,1.0,1.0,1.0,1.0 +dense,690,10,9.57229,0.0,0.0,0.0,0.0 +dense,691,10,9.221,0.43068,1.0,0.25,0.0 +dense,692,10,9.23163,1.0,1.0,1.0,1.0 +dense,693,10,9.39579,1.0,1.0,1.0,1.0 +dense,700,10,9.13404,0.63093,1.0,0.5,1.0 +dense,702,10,8.82663,0.63093,1.0,0.5,1.0 +dense,715,10,8.82558,0.0,0.0,0.0,0.0 +dense,716,10,10.54804,0.0,0.0,0.0,0.0 +dense,718,10,9.31846,1.0,1.0,1.0,1.0 +dense,721,10,9.39229,1.0,1.0,1.0,1.0 +dense,723,10,9.67475,1.0,1.0,1.0,1.0 +dense,727,10,9.16004,0.43068,1.0,0.25,0.0 +dense,728,10,9.12346,0.65092,1.0,0.5,1.0 +dense,729,10,8.94025,1.0,1.0,1.0,1.0 +dense,742,10,9.33454,0.30103,1.0,0.11111,0.0 +dense,743,10,9.08408,0.0,0.0,0.0,0.0 +dense,744,10,9.26138,0.5,1.0,0.33333,1.0 +dense,756,10,8.833,0.63093,1.0,0.5,1.0 +dense,759,10,10.05567,1.0,1.0,1.0,1.0 +dense,768,10,10.24592,0.0,0.0,0.0,0.0 +dense,770,10,10.33617,1.0,1.0,1.0,1.0 +dense,775,10,9.57192,0.0,0.0,0.0,0.0 +dense,781,10,9.043,0.63093,1.0,0.5,1.0 +dense,783,10,9.39675,0.38685,1.0,0.2,0.0 +dense,784,10,8.95208,0.63093,1.0,0.5,1.0 +dense,785,10,8.6995,0.0,0.0,0.0,0.0 +dense,793,10,10.12779,1.0,1.0,1.0,1.0 +dense,800,10,9.748,0.63093,1.0,0.5,1.0 +dense,805,10,10.29742,1.0,1.0,1.0,1.0 +dense,808,10,9.44617,1.0,1.0,1.0,1.0 +dense,811,10,9.04804,1.0,1.0,1.0,1.0 +dense,814,10,11.33271,1.0,1.0,1.0,1.0 +dense,820,10,9.2015,0.0,0.0,0.0,0.0 +dense,821,10,9.38367,0.0,0.0,0.0,0.0 +dense,823,10,9.40279,1.0,1.0,1.0,1.0 +dense,830,10,10.61463,0.0,0.0,0.0,0.0 +dense,831,10,9.62538,0.0,0.0,0.0,0.0 +dense,832,10,9.34396,1.0,1.0,1.0,1.0 +dense,834,10,9.33867,0.0,0.0,0.0,0.0 +dense,837,10,9.7265,0.63093,1.0,0.5,1.0 +dense,839,10,13.53763,0.5,1.0,0.33333,1.0 +dense,845,10,10.75008,0.43068,1.0,0.25,0.0 +dense,847,10,11.44754,1.0,1.0,1.0,1.0 +dense,852,10,9.458,0.63093,1.0,0.5,1.0 +dense,859,10,10.22104,0.43068,1.0,0.25,0.0 +dense,870,10,9.05288,0.0,0.0,0.0,0.0 +dense,873,10,9.64763,0.26762,0.4,0.33333,1.0 +dense,879,10,9.59429,1.0,1.0,1.0,1.0 +dense,880,10,9.73196,0.63093,1.0,0.5,1.0 +dense,882,10,8.99317,1.0,1.0,1.0,1.0 +dense,887,10,9.29692,0.0,0.0,0.0,0.0 +dense,903,10,9.51708,1.0,1.0,1.0,1.0 +dense,904,10,10.32637,1.0,1.0,1.0,1.0 +dense,907,10,9.85862,1.0,1.0,1.0,1.0 +dense,911,10,9.03888,1.0,1.0,1.0,1.0 +dense,913,10,11.14817,0.31546,1.0,0.125,0.0 +dense,914,10,9.95054,0.43068,1.0,0.25,0.0 +dense,921,10,10.45258,1.0,1.0,1.0,1.0 +dense,922,10,11.337,1.0,1.0,1.0,1.0 +dense,936,10,12.71867,1.0,1.0,1.0,1.0 +dense,956,10,14.41079,1.0,1.0,1.0,1.0 +dense,957,10,11.19117,0.5,1.0,0.33333,1.0 +dense,960,10,9.11888,1.0,1.0,1.0,1.0 +dense,967,10,10.498,1.0,1.0,1.0,1.0 +dense,971,10,9.48108,0.73157,1.0,0.5,1.0 +dense,975,10,10.276,0.0,0.0,0.0,0.0 +dense,982,10,9.26346,1.0,1.0,1.0,1.0 +dense,985,10,9.19404,1.0,1.0,1.0,1.0 +dense,993,10,9.85346,0.63093,1.0,0.5,1.0 +dense,1012,10,9.73392,1.0,1.0,1.0,1.0 +dense,1014,10,10.69617,1.0,1.0,1.0,1.0 +dense,1019,10,10.93546,0.43068,1.0,0.25,0.0 +dense,1020,10,9.62887,0.63093,1.0,0.5,1.0 +dense,1021,10,9.44629,0.63093,1.0,0.5,1.0 +dense,1024,10,11.47979,0.0,0.0,0.0,0.0 +dense,1029,10,10.41608,0.96747,1.0,1.0,1.0 +dense,1041,10,10.34496,0.0,0.0,0.0,0.0 +dense,1049,10,10.06758,0.0,0.0,0.0,0.0 +dense,1062,10,10.55487,0.63093,1.0,0.5,1.0 +dense,1086,10,10.00437,1.0,1.0,1.0,1.0 +dense,1088,10,10.55429,1.0,1.0,1.0,1.0 +dense,1089,10,9.49404,1.0,1.0,1.0,1.0 +dense,1099,10,8.83654,0.5,1.0,0.33333,1.0 +dense,1100,10,8.27146,0.43068,1.0,0.25,0.0 +dense,1104,10,10.27875,1.0,1.0,1.0,1.0 +dense,1107,10,10.08379,0.5,1.0,0.33333,1.0 +dense,1110,10,9.19833,0.0,0.0,0.0,0.0 +dense,1121,10,8.96383,1.0,1.0,1.0,1.0 +dense,1130,10,9.69629,1.0,1.0,1.0,1.0 +dense,1132,10,8.97925,0.2372,0.5,0.2,0.0 +dense,1137,10,9.61417,1.0,1.0,1.0,1.0 +dense,1140,10,9.05938,1.0,1.0,1.0,1.0 +dense,1144,10,9.41842,1.0,1.0,1.0,1.0 +dense,1146,10,9.73996,1.0,1.0,1.0,1.0 +dense,1150,10,9.16467,1.0,1.0,1.0,1.0 +dense,1163,10,8.88825,1.0,1.0,1.0,1.0 +dense,1175,10,8.99779,0.0,0.0,0.0,0.0 +dense,1179,10,9.07746,0.0,0.0,0.0,0.0 +dense,1180,10,8.65729,0.0,0.0,0.0,0.0 +dense,1185,10,11.13533,0.5,1.0,0.33333,1.0 +dense,1187,10,9.71167,0.63093,1.0,0.5,1.0 +dense,1191,10,8.96042,0.63093,1.0,0.5,1.0 +dense,1194,10,9.35987,1.0,1.0,1.0,1.0 +dense,1196,10,9.54392,0.38685,1.0,0.2,0.0 +dense,1197,10,9.57471,0.38685,1.0,0.2,0.0 +dense,1199,10,9.10854,0.0,0.0,0.0,0.0 +dense,1200,10,10.15933,0.0,0.0,0.0,0.0 +dense,1202,10,9.11096,0.63093,1.0,0.5,1.0 +dense,1204,10,9.69817,1.0,1.0,1.0,1.0 +dense,1207,10,10.71412,1.0,1.0,1.0,1.0 +dense,1213,10,9.67179,0.0,0.0,0.0,0.0 +dense,1216,10,9.92546,1.0,1.0,1.0,1.0 +dense,1221,10,9.09579,0.31546,1.0,0.125,0.0 +dense,1225,10,9.72125,1.0,1.0,1.0,1.0 +dense,1226,10,10.08121,0.0,0.0,0.0,0.0 +dense,1232,10,8.90667,1.0,1.0,1.0,1.0 +dense,1241,10,8.74358,1.0,1.0,1.0,1.0 +dense,1245,10,9.88354,0.30103,1.0,0.11111,0.0 +dense,1259,10,10.11171,0.63093,1.0,0.5,1.0 +dense,1262,10,9.00242,1.0,1.0,1.0,1.0 +dense,1266,10,10.32563,0.38685,1.0,0.2,0.0 +dense,1270,10,9.13229,1.0,1.0,1.0,1.0 +dense,1271,10,9.87979,1.0,1.0,1.0,1.0 +dense,1272,10,9.02512,0.0,0.0,0.0,0.0 +dense,1273,10,9.14375,1.0,1.0,1.0,1.0 +dense,1274,10,9.65025,0.9469,1.0,1.0,1.0 +dense,1278,10,9.49913,1.0,1.0,1.0,1.0 +dense,1279,10,8.79987,1.0,1.0,1.0,1.0 +dense,1280,10,9.60288,0.0,0.0,0.0,0.0 +dense,1281,10,9.28608,0.31546,1.0,0.125,0.0 +dense,1282,10,9.50971,1.0,1.0,1.0,1.0 +dense,1290,10,9.42429,1.0,1.0,1.0,1.0 +dense,1292,10,9.67167,1.0,1.0,1.0,1.0 +dense,1298,10,9.51771,1.0,1.0,1.0,1.0 +dense,1303,10,9.60367,1.0,1.0,1.0,1.0 +dense,1316,10,9.76221,0.0,0.0,0.0,0.0 +dense,1319,10,9.31358,1.0,1.0,1.0,1.0 +dense,1320,10,8.95312,1.0,1.0,1.0,1.0 +dense,1332,10,9.83483,0.0,0.0,0.0,0.0 +dense,1335,10,9.21542,1.0,1.0,1.0,1.0 +dense,1336,10,10.11729,1.0,1.0,1.0,1.0 +dense,1337,10,9.89596,0.28906,1.0,0.1,0.0 +dense,1339,10,9.29675,1.0,1.0,1.0,1.0 +dense,1344,10,9.75092,0.63093,1.0,0.5,1.0 +dense,1352,10,9.35767,1.0,1.0,1.0,1.0 +dense,1359,10,9.50817,1.0,1.0,1.0,1.0 +dense,1362,10,8.99062,0.0,0.0,0.0,0.0 +dense,1363,10,9.18075,0.0,0.0,0.0,0.0 +dense,1368,10,9.34963,0.0,0.0,0.0,0.0 +dense,1370,10,8.71604,0.5,1.0,0.33333,1.0 +dense,1379,10,9.36408,0.95583,1.0,1.0,1.0 +dense,1382,10,9.42504,1.0,1.0,1.0,1.0 +dense,1385,10,9.00183,0.63093,1.0,0.5,1.0 +dense,1389,10,9.63775,1.0,1.0,1.0,1.0 +dense,1395,10,10.01796,0.5,1.0,0.33333,1.0 +bm25,1,10,5.02104,0.0,0.0,0.0,0.0 +bm25,3,10,16.10962,0.63093,1.0,0.5,1.0 +bm25,5,10,6.61529,1.0,1.0,1.0,1.0 +bm25,13,10,9.02621,0.0,0.0,0.0,0.0 +bm25,36,10,10.14104,0.0,0.0,0.0,0.0 +bm25,42,10,12.69625,1.0,1.0,1.0,1.0 +bm25,48,10,12.12929,0.5,1.0,0.33333,1.0 +bm25,49,10,5.87804,1.0,1.0,1.0,1.0 +bm25,50,10,7.06804,1.0,1.0,1.0,1.0 +bm25,51,10,8.85283,1.0,1.0,1.0,1.0 +bm25,53,10,9.27504,1.0,1.0,1.0,1.0 +bm25,54,10,10.36696,0.0,0.0,0.0,0.0 +bm25,56,10,11.88525,1.0,1.0,1.0,1.0 +bm25,57,10,12.41467,1.0,1.0,1.0,1.0 +bm25,70,10,6.15571,1.0,1.0,1.0,1.0 +bm25,72,10,6.86937,0.5,1.0,0.33333,1.0 +bm25,75,10,12.24871,0.63093,1.0,0.5,1.0 +bm25,94,10,6.72263,0.0,0.0,0.0,0.0 +bm25,99,10,9.47204,1.0,1.0,1.0,1.0 +bm25,100,10,7.18354,1.0,1.0,1.0,1.0 +bm25,113,10,12.35,0.63093,1.0,0.5,1.0 +bm25,115,10,9.70908,0.63093,1.0,0.5,1.0 +bm25,118,10,10.08258,1.0,1.0,1.0,1.0 +bm25,124,10,11.41567,1.0,1.0,1.0,1.0 +bm25,127,10,11.41371,0.0,0.0,0.0,0.0 +bm25,128,10,7.05846,0.0,0.0,0.0,0.0 +bm25,129,10,13.03096,1.0,1.0,1.0,1.0 +bm25,130,10,13.70837,1.0,1.0,1.0,1.0 +bm25,132,10,5.46787,0.0,0.0,0.0,0.0 +bm25,133,10,15.60187,0.32095,0.6,0.16667,0.0 +bm25,137,10,10.91925,1.0,1.0,1.0,1.0 +bm25,141,10,9.63342,0.91972,1.0,1.0,1.0 +bm25,142,10,17.07158,1.0,1.0,1.0,1.0 +bm25,143,10,16.56092,1.0,1.0,1.0,1.0 +bm25,146,10,15.92083,1.0,1.0,1.0,1.0 +bm25,148,10,6.59904,1.0,1.0,1.0,1.0 +bm25,163,10,10.38608,0.63093,1.0,0.5,1.0 +bm25,171,10,11.39996,1.0,1.0,1.0,1.0 +bm25,179,10,7.01017,0.12315,0.25,0.125,0.0 +bm25,180,10,16.18529,1.0,1.0,1.0,1.0 +bm25,183,10,7.38754,0.38685,1.0,0.2,0.0 +bm25,185,10,8.93542,0.43068,1.0,0.25,0.0 +bm25,198,10,5.07875,0.0,0.0,0.0,0.0 +bm25,208,10,6.92583,1.0,1.0,1.0,1.0 +bm25,212,10,7.00362,1.0,1.0,1.0,1.0 +bm25,213,10,11.82817,0.38685,1.0,0.2,0.0 +bm25,216,10,10.95425,1.0,1.0,1.0,1.0 +bm25,217,10,10.43204,1.0,1.0,1.0,1.0 +bm25,218,10,9.41396,1.0,1.0,1.0,1.0 +bm25,219,10,9.50104,1.0,1.0,1.0,1.0 +bm25,230,10,14.27471,1.0,1.0,1.0,1.0 +bm25,232,10,14.42071,0.63093,1.0,0.5,1.0 +bm25,233,10,13.89271,0.5,1.0,0.33333,1.0 +bm25,236,10,10.35388,1.0,1.0,1.0,1.0 +bm25,237,10,11.11283,0.5,1.0,0.33333,1.0 +bm25,238,10,7.34821,0.0,0.0,0.0,0.0 +bm25,239,10,8.67125,0.0,0.0,0.0,0.0 +bm25,248,10,9.62742,1.0,1.0,1.0,1.0 +bm25,249,10,6.99225,1.0,1.0,1.0,1.0 +bm25,261,10,11.54746,0.30657,0.5,0.33333,1.0 +bm25,268,10,5.35275,0.43068,1.0,0.25,0.0 +bm25,269,10,6.19833,0.5,1.0,0.33333,1.0 +bm25,274,10,20.68396,1.0,1.0,1.0,1.0 +bm25,275,10,14.4565,0.46928,0.33333,1.0,1.0 +bm25,279,10,11.52517,1.0,1.0,1.0,1.0 +bm25,294,10,13.74958,0.0,0.0,0.0,0.0 +bm25,295,10,23.51008,1.0,1.0,1.0,1.0 +bm25,298,10,18.417,1.0,1.0,1.0,1.0 +bm25,300,10,25.592,0.0,0.0,0.0,0.0 +bm25,303,10,12.64154,0.43068,1.0,0.25,0.0 +bm25,312,10,11.51921,0.30103,1.0,0.11111,0.0 +bm25,314,10,16.93296,0.5,1.0,0.33333,1.0 +bm25,324,10,4.30133,0.0,0.0,0.0,0.0 +bm25,327,10,9.28308,1.0,1.0,1.0,1.0 +bm25,338,10,5.64962,1.0,1.0,1.0,1.0 +bm25,343,10,14.01117,0.57064,1.0,0.33333,1.0 +bm25,350,10,11.4735,0.63093,1.0,0.5,1.0 +bm25,354,10,9.56746,1.0,1.0,1.0,1.0 +bm25,362,10,19.46746,1.0,1.0,1.0,1.0 +bm25,380,10,10.73308,1.0,1.0,1.0,1.0 +bm25,384,10,11.15062,0.0,0.0,0.0,0.0 +bm25,385,10,10.46054,0.62405,1.0,0.5,1.0 +bm25,386,10,13.07983,1.0,1.0,1.0,1.0 +bm25,388,10,8.69504,0.5,1.0,0.33333,1.0 +bm25,399,10,9.95933,1.0,1.0,1.0,1.0 +bm25,410,10,9.42967,1.0,1.0,1.0,1.0 +bm25,411,10,8.97554,1.0,1.0,1.0,1.0 +bm25,415,10,10.32767,0.31546,1.0,0.125,0.0 +bm25,421,10,10.32025,0.0,0.0,0.0,0.0 +bm25,431,10,10.73333,0.0,0.0,0.0,0.0 +bm25,436,10,10.99133,1.0,1.0,1.0,1.0 +bm25,437,10,16.66796,0.0,0.0,0.0,0.0 +bm25,439,10,12.00158,0.0,0.0,0.0,0.0 +bm25,440,10,10.99375,0.0,0.0,0.0,0.0 +bm25,443,10,8.30275,1.0,1.0,1.0,1.0 +bm25,452,10,8.78629,0.19343,0.5,0.125,0.0 +bm25,475,10,10.08129,0.33333,1.0,0.14286,0.0 +bm25,478,10,20.49008,0.0,0.0,0.0,0.0 +bm25,491,10,11.53675,1.0,1.0,1.0,1.0 +bm25,501,10,7.27008,0.33333,1.0,0.14286,0.0 +bm25,502,10,13.50767,0.0,0.0,0.0,0.0 +bm25,507,10,12.21292,0.63093,1.0,0.5,1.0 +bm25,508,10,9.83792,0.0,0.0,0.0,0.0 +bm25,513,10,6.93387,0.63093,1.0,0.5,1.0 +bm25,514,10,15.93187,0.0,0.0,0.0,0.0 +bm25,516,10,11.90146,1.0,1.0,1.0,1.0 +bm25,517,10,6.40633,0.0,0.0,0.0,0.0 +bm25,521,10,19.97596,1.0,1.0,1.0,1.0 +bm25,525,10,16.79567,1.0,1.0,1.0,1.0 +bm25,527,10,13.79612,0.63093,1.0,0.5,1.0 +bm25,528,10,18.75946,1.0,1.0,1.0,1.0 +bm25,532,10,5.94421,1.0,1.0,1.0,1.0 +bm25,533,10,5.55604,1.0,1.0,1.0,1.0 +bm25,535,10,8.53521,0.0,0.0,0.0,0.0 +bm25,536,10,5.86271,0.0,0.0,0.0,0.0 +bm25,539,10,5.88562,1.0,1.0,1.0,1.0 +bm25,540,10,7.56342,0.38685,0.5,0.5,1.0 +bm25,544,10,8.02296,0.0,0.0,0.0,0.0 +bm25,549,10,6.46258,1.0,1.0,1.0,1.0 +bm25,551,10,21.93738,0.35621,1.0,0.16667,0.0 +bm25,552,10,17.50737,1.0,1.0,1.0,1.0 +bm25,554,10,12.31558,0.38685,1.0,0.2,0.0 +bm25,560,10,11.38021,0.0,0.0,0.0,0.0 +bm25,569,10,9.60021,1.0,1.0,1.0,1.0 +bm25,575,10,14.54058,0.0,0.0,0.0,0.0 +bm25,577,10,21.57025,0.0,0.0,0.0,0.0 +bm25,578,10,8.77196,1.0,1.0,1.0,1.0 +bm25,587,10,24.99725,0.0,0.0,0.0,0.0 +bm25,589,10,17.98575,1.0,1.0,1.0,1.0 +bm25,593,10,10.17729,1.0,1.0,1.0,1.0 +bm25,597,10,7.14267,0.88546,1.0,1.0,1.0 +bm25,598,10,19.60375,0.38685,1.0,0.2,0.0 +bm25,613,10,11.60567,1.0,1.0,1.0,1.0 +bm25,619,10,13.19783,0.26407,0.5,0.25,0.0 +bm25,623,10,10.81379,0.0,0.0,0.0,0.0 +bm25,628,10,13.15508,0.33333,1.0,0.14286,0.0 +bm25,636,10,7.91179,0.38685,1.0,0.2,0.0 +bm25,637,10,11.4995,0.33333,1.0,0.14286,0.0 +bm25,641,10,8.549,0.61315,0.5,1.0,1.0 +bm25,644,10,6.16375,0.63093,1.0,0.5,1.0 +bm25,649,10,10.29242,0.33333,1.0,0.14286,0.0 +bm25,659,10,6.43842,0.0,0.0,0.0,0.0 +bm25,660,10,5.22396,0.0,0.0,0.0,0.0 +bm25,674,10,10.51621,1.0,1.0,1.0,1.0 +bm25,684,10,9.70437,0.63093,1.0,0.5,1.0 +bm25,690,10,17.32883,0.0,0.0,0.0,0.0 +bm25,691,10,10.95983,1.0,1.0,1.0,1.0 +bm25,692,10,9.01617,0.31546,1.0,0.125,0.0 +bm25,693,10,8.58704,0.33333,1.0,0.14286,0.0 +bm25,700,10,9.78167,0.63093,1.0,0.5,1.0 +bm25,702,10,9.84279,0.31546,1.0,0.125,0.0 +bm25,715,10,14.15738,0.0,0.0,0.0,0.0 +bm25,716,10,10.13142,0.0,0.0,0.0,0.0 +bm25,718,10,8.77058,0.33333,1.0,0.14286,0.0 +bm25,721,10,11.01337,1.0,1.0,1.0,1.0 +bm25,723,10,12.66137,1.0,1.0,1.0,1.0 +bm25,727,10,11.39992,0.0,0.0,0.0,0.0 +bm25,728,10,10.09354,0.80657,1.0,1.0,1.0 +bm25,729,10,9.97425,0.63093,1.0,0.5,1.0 +bm25,742,10,6.49692,1.0,1.0,1.0,1.0 +bm25,743,10,3.5305,1.0,1.0,1.0,1.0 +bm25,744,10,13.04967,0.63093,1.0,0.5,1.0 +bm25,756,10,13.20254,0.0,0.0,0.0,0.0 +bm25,759,10,15.40096,1.0,1.0,1.0,1.0 +bm25,768,10,9.79317,0.0,0.0,0.0,0.0 +bm25,770,10,23.25229,1.0,1.0,1.0,1.0 +bm25,775,10,13.56971,0.0,0.0,0.0,0.0 +bm25,781,10,11.92079,0.43068,1.0,0.25,0.0 +bm25,783,10,11.67925,0.0,0.0,0.0,0.0 +bm25,784,10,13.35312,0.63093,1.0,0.5,1.0 +bm25,785,10,14.35275,0.63093,1.0,0.5,1.0 +bm25,793,10,5.24975,1.0,1.0,1.0,1.0 +bm25,800,10,15.33183,0.0,0.0,0.0,0.0 +bm25,805,10,6.11533,1.0,1.0,1.0,1.0 +bm25,808,10,8.36138,0.63093,1.0,0.5,1.0 +bm25,811,10,11.984,1.0,1.0,1.0,1.0 +bm25,814,10,19.30812,1.0,1.0,1.0,1.0 +bm25,820,10,6.68796,0.0,0.0,0.0,0.0 +bm25,821,10,6.40971,0.0,0.0,0.0,0.0 +bm25,823,10,6.27604,1.0,1.0,1.0,1.0 +bm25,830,10,13.25783,0.0,0.0,0.0,0.0 +bm25,831,10,10.09417,0.0,0.0,0.0,0.0 +bm25,832,10,5.34204,1.0,1.0,1.0,1.0 +bm25,834,10,9.67038,0.33333,1.0,0.14286,0.0 +bm25,837,10,7.47521,1.0,1.0,1.0,1.0 +bm25,839,10,11.86112,0.5,1.0,0.33333,1.0 +bm25,845,10,7.67575,1.0,1.0,1.0,1.0 +bm25,847,10,14.48383,1.0,1.0,1.0,1.0 +bm25,852,10,12.94467,0.63093,1.0,0.5,1.0 +bm25,859,10,6.75488,0.63093,1.0,0.5,1.0 +bm25,870,10,3.3905,0.0,0.0,0.0,0.0 +bm25,873,10,7.05704,0.0,0.0,0.0,0.0 +bm25,879,10,8.94933,1.0,1.0,1.0,1.0 +bm25,880,10,7.33754,1.0,1.0,1.0,1.0 +bm25,882,10,7.823,0.63093,1.0,0.5,1.0 +bm25,887,10,12.15438,0.0,0.0,0.0,0.0 +bm25,903,10,7.97412,1.0,1.0,1.0,1.0 +bm25,904,10,17.32929,1.0,1.0,1.0,1.0 +bm25,907,10,14.08487,1.0,1.0,1.0,1.0 +bm25,911,10,11.55475,1.0,1.0,1.0,1.0 +bm25,913,10,5.93296,0.0,0.0,0.0,0.0 +bm25,914,10,6.07092,0.0,0.0,0.0,0.0 +bm25,921,10,8.10558,0.43068,1.0,0.25,0.0 +bm25,922,10,10.38783,1.0,1.0,1.0,1.0 +bm25,936,10,6.84962,1.0,1.0,1.0,1.0 +bm25,956,10,11.36442,0.63093,1.0,0.5,1.0 +bm25,957,10,9.76367,0.5,1.0,0.33333,1.0 +bm25,960,10,4.39825,1.0,1.0,1.0,1.0 +bm25,967,10,7.89654,0.69343,1.0,0.5,1.0 +bm25,971,10,16.97829,0.64446,1.0,0.33333,1.0 +bm25,975,10,7.50396,0.0,0.0,0.0,0.0 +bm25,982,10,15.5795,1.0,1.0,1.0,1.0 +bm25,985,10,11.21996,1.0,1.0,1.0,1.0 +bm25,993,10,9.92417,0.5,1.0,0.33333,1.0 +bm25,1012,10,7.61842,1.0,1.0,1.0,1.0 +bm25,1014,10,9.47104,1.0,1.0,1.0,1.0 +bm25,1019,10,7.44254,0.63093,1.0,0.5,1.0 +bm25,1020,10,17.8105,0.5,1.0,0.33333,1.0 +bm25,1021,10,20.73158,0.5,1.0,0.33333,1.0 +bm25,1024,10,9.47762,1.0,1.0,1.0,1.0 +bm25,1029,10,17.35683,0.76536,0.66667,1.0,1.0 +bm25,1041,10,13.39667,0.41183,1.0,0.16667,0.0 +bm25,1049,10,9.07212,0.0,0.0,0.0,0.0 +bm25,1062,10,4.84004,1.0,1.0,1.0,1.0 +bm25,1086,10,15.71363,1.0,1.0,1.0,1.0 +bm25,1088,10,10.6375,0.0,0.0,0.0,0.0 +bm25,1089,10,13.75325,1.0,1.0,1.0,1.0 +bm25,1099,10,3.51079,0.0,0.0,0.0,0.0 +bm25,1100,10,3.27188,0.0,0.0,0.0,0.0 +bm25,1104,10,22.68571,1.0,1.0,1.0,1.0 +bm25,1107,10,8.28267,1.0,1.0,1.0,1.0 +bm25,1110,10,8.02971,0.0,0.0,0.0,0.0 +bm25,1121,10,10.59975,0.38685,1.0,0.2,0.0 +bm25,1130,10,13.90579,1.0,1.0,1.0,1.0 +bm25,1132,10,11.91846,0.0,0.0,0.0,0.0 +bm25,1137,10,8.67542,0.0,0.0,0.0,0.0 +bm25,1140,10,10.92612,0.30103,1.0,0.11111,0.0 +bm25,1144,10,18.85158,1.0,1.0,1.0,1.0 +bm25,1146,10,10.92546,1.0,1.0,1.0,1.0 +bm25,1150,10,11.66167,1.0,1.0,1.0,1.0 +bm25,1163,10,10.19796,1.0,1.0,1.0,1.0 +bm25,1175,10,7.95842,0.0,0.0,0.0,0.0 +bm25,1179,10,11.12475,0.0,0.0,0.0,0.0 +bm25,1180,10,9.48658,0.28906,1.0,0.1,0.0 +bm25,1185,10,26.67521,1.0,1.0,1.0,1.0 +bm25,1187,10,23.71529,1.0,1.0,1.0,1.0 +bm25,1191,10,10.41983,0.0,0.0,0.0,0.0 +bm25,1194,10,20.54817,0.0,0.0,0.0,0.0 +bm25,1196,10,12.18038,0.0,0.0,0.0,0.0 +bm25,1197,10,13.77562,0.0,0.0,0.0,0.0 +bm25,1199,10,16.18321,0.0,0.0,0.0,0.0 +bm25,1200,10,21.61933,0.0,0.0,0.0,0.0 +bm25,1202,10,14.239,0.0,0.0,0.0,0.0 +bm25,1204,10,13.75129,1.0,1.0,1.0,1.0 +bm25,1207,10,17.29383,0.35621,1.0,0.16667,0.0 +bm25,1213,10,12.81813,0.0,0.0,0.0,0.0 +bm25,1216,10,11.15983,1.0,1.0,1.0,1.0 +bm25,1221,10,15.0115,0.0,0.0,0.0,0.0 +bm25,1225,10,7.56033,1.0,1.0,1.0,1.0 +bm25,1226,10,15.0295,0.0,0.0,0.0,0.0 +bm25,1232,10,13.17204,1.0,1.0,1.0,1.0 +bm25,1241,10,8.33033,0.0,0.0,0.0,0.0 +bm25,1245,10,9.92354,0.0,0.0,0.0,0.0 +bm25,1259,10,18.17137,0.33333,1.0,0.14286,0.0 +bm25,1262,10,9.94996,0.63093,1.0,0.5,1.0 +bm25,1266,10,21.84096,0.63093,1.0,0.5,1.0 +bm25,1270,10,11.94404,1.0,1.0,1.0,1.0 +bm25,1271,10,16.55346,1.0,1.0,1.0,1.0 +bm25,1272,10,10.46617,0.0,0.0,0.0,0.0 +bm25,1273,10,10.57417,1.0,1.0,1.0,1.0 +bm25,1274,10,20.18742,0.71226,1.0,0.5,1.0 +bm25,1278,10,13.38575,0.0,0.0,0.0,0.0 +bm25,1279,10,11.68337,0.0,0.0,0.0,0.0 +bm25,1280,10,12.35929,0.0,0.0,0.0,0.0 +bm25,1281,10,9.50821,0.0,0.0,0.0,0.0 +bm25,1282,10,12.869,0.35621,1.0,0.16667,0.0 +bm25,1290,10,9.36837,1.0,1.0,1.0,1.0 +bm25,1292,10,9.74112,1.0,1.0,1.0,1.0 +bm25,1298,10,19.31408,1.0,1.0,1.0,1.0 +bm25,1303,10,6.09479,0.0,0.0,0.0,0.0 +bm25,1316,10,8.11121,0.0,0.0,0.0,0.0 +bm25,1319,10,8.89904,1.0,1.0,1.0,1.0 +bm25,1320,10,14.01721,1.0,1.0,1.0,1.0 +bm25,1332,10,13.21167,0.0,0.0,0.0,0.0 +bm25,1335,10,8.11525,1.0,1.0,1.0,1.0 +bm25,1336,10,6.30163,1.0,1.0,1.0,1.0 +bm25,1337,10,9.46729,1.0,1.0,1.0,1.0 +bm25,1339,10,13.09654,1.0,1.0,1.0,1.0 +bm25,1344,10,21.95104,0.63093,1.0,0.5,1.0 +bm25,1352,10,9.85783,1.0,1.0,1.0,1.0 +bm25,1359,10,16.778,1.0,1.0,1.0,1.0 +bm25,1362,10,6.74,0.0,0.0,0.0,0.0 +bm25,1363,10,9.49342,0.0,0.0,0.0,0.0 +bm25,1368,10,6.8545,0.0,0.0,0.0,0.0 +bm25,1370,10,8.11725,0.0,0.0,0.0,0.0 +bm25,1379,10,12.93425,0.41443,0.5,0.5,1.0 +bm25,1382,10,8.71808,0.0,0.0,0.0,0.0 +bm25,1385,10,4.69367,1.0,1.0,1.0,1.0 +bm25,1389,10,6.69988,1.0,1.0,1.0,1.0 +bm25,1395,10,19.72512,0.0,0.0,0.0,0.0 +dense+rerank,1,10,1123.79142,0.0,0.0,0.0,0.0 +dense+rerank,3,10,1032.48267,1.0,1.0,1.0,1.0 +dense+rerank,5,10,1109.01967,1.0,1.0,1.0,1.0 +dense+rerank,13,10,1103.68925,0.0,0.0,0.0,0.0 +dense+rerank,36,10,1077.97867,0.38896,1.0,0.14286,0.0 +dense+rerank,42,10,1074.05229,1.0,1.0,1.0,1.0 +dense+rerank,48,10,1106.43521,0.0,0.0,0.0,0.0 +dense+rerank,49,10,1076.86192,1.0,1.0,1.0,1.0 +dense+rerank,50,10,1152.32192,1.0,1.0,1.0,1.0 +dense+rerank,51,10,1114.718,1.0,1.0,1.0,1.0 +dense+rerank,53,10,1123.31592,1.0,1.0,1.0,1.0 +dense+rerank,54,10,1080.08425,0.35621,1.0,0.16667,0.0 +dense+rerank,56,10,1100.64671,1.0,1.0,1.0,1.0 +dense+rerank,57,10,1101.39783,1.0,1.0,1.0,1.0 +dense+rerank,70,10,1151.23329,0.0,0.0,0.0,0.0 +dense+rerank,72,10,1153.25692,1.0,1.0,1.0,1.0 +dense+rerank,75,10,1112.48379,1.0,1.0,1.0,1.0 +dense+rerank,94,10,1106.95029,0.31546,1.0,0.125,0.0 +dense+rerank,99,10,1137.70262,1.0,1.0,1.0,1.0 +dense+rerank,100,10,975.53646,1.0,1.0,1.0,1.0 +dense+rerank,113,10,1026.49671,1.0,1.0,1.0,1.0 +dense+rerank,115,10,1140.12192,1.0,1.0,1.0,1.0 +dense+rerank,118,10,1131.09175,1.0,1.0,1.0,1.0 +dense+rerank,124,10,1105.00454,1.0,1.0,1.0,1.0 +dense+rerank,127,10,1150.1555,0.63093,1.0,0.5,1.0 +dense+rerank,128,10,1114.48542,0.0,0.0,0.0,0.0 +dense+rerank,129,10,1101.4235,1.0,1.0,1.0,1.0 +dense+rerank,130,10,1165.80713,1.0,1.0,1.0,1.0 +dense+rerank,132,10,1175.55933,0.0,0.0,0.0,0.0 +dense+rerank,133,10,1151.89404,0.79725,0.8,1.0,1.0 +dense+rerank,137,10,1145.46288,1.0,1.0,1.0,1.0 +dense+rerank,141,10,1141.78171,0.91972,1.0,1.0,1.0 +dense+rerank,142,10,1142.21338,1.0,1.0,1.0,1.0 +dense+rerank,143,10,1146.65992,1.0,1.0,1.0,1.0 +dense+rerank,146,10,1125.278,1.0,1.0,1.0,1.0 +dense+rerank,148,10,1141.69921,1.0,1.0,1.0,1.0 +dense+rerank,163,10,1117.48492,0.63093,1.0,0.5,1.0 +dense+rerank,171,10,1157.83225,1.0,1.0,1.0,1.0 +dense+rerank,179,10,1168.137,0.90522,1.0,1.0,1.0 +dense+rerank,180,10,1151.58221,1.0,1.0,1.0,1.0 +dense+rerank,183,10,1079.27425,1.0,1.0,1.0,1.0 +dense+rerank,185,10,1134.59917,0.43068,1.0,0.25,0.0 +dense+rerank,198,10,1135.96313,0.0,0.0,0.0,0.0 +dense+rerank,208,10,1155.74363,1.0,1.0,1.0,1.0 +dense+rerank,212,10,1184.41038,0.5,1.0,0.33333,1.0 +dense+rerank,213,10,1152.37488,0.0,0.0,0.0,0.0 +dense+rerank,216,10,774.91633,1.0,1.0,1.0,1.0 +dense+rerank,217,10,1160.19504,1.0,1.0,1.0,1.0 +dense+rerank,218,10,1015.80796,1.0,1.0,1.0,1.0 +dense+rerank,219,10,1094.69896,1.0,1.0,1.0,1.0 +dense+rerank,230,10,1180.37588,1.0,1.0,1.0,1.0 +dense+rerank,232,10,1180.30567,0.63093,1.0,0.5,1.0 +dense+rerank,233,10,1129.36187,1.0,1.0,1.0,1.0 +dense+rerank,236,10,1187.07254,1.0,1.0,1.0,1.0 +dense+rerank,237,10,1073.7945,0.43068,1.0,0.25,0.0 +dense+rerank,238,10,1180.71437,0.0,0.0,0.0,0.0 +dense+rerank,239,10,1233.15046,0.0,0.0,0.0,0.0 +dense+rerank,248,10,1179.48346,1.0,1.0,1.0,1.0 +dense+rerank,249,10,1096.71629,1.0,1.0,1.0,1.0 +dense+rerank,261,10,1200.18354,0.38685,0.5,0.5,1.0 +dense+rerank,268,10,1163.59446,0.63093,1.0,0.5,1.0 +dense+rerank,269,10,1208.72838,0.63093,1.0,0.5,1.0 +dense+rerank,274,10,1161.69717,1.0,1.0,1.0,1.0 +dense+rerank,275,10,1217.37071,0.87108,1.0,1.0,1.0 +dense+rerank,279,10,1174.43017,1.0,1.0,1.0,1.0 +dense+rerank,294,10,763.11954,1.0,1.0,1.0,1.0 +dense+rerank,295,10,1186.82517,1.0,1.0,1.0,1.0 +dense+rerank,298,10,1188.27658,1.0,1.0,1.0,1.0 +dense+rerank,300,10,806.29587,0.5,1.0,0.33333,1.0 +dense+rerank,303,10,1223.99888,0.0,0.0,0.0,0.0 +dense+rerank,312,10,1057.40067,0.0,0.0,0.0,0.0 +dense+rerank,314,10,1203.28383,0.0,0.0,0.0,0.0 +dense+rerank,324,10,1168.70367,0.38685,1.0,0.2,0.0 +dense+rerank,327,10,1164.12988,1.0,1.0,1.0,1.0 +dense+rerank,338,10,1210.88517,1.0,1.0,1.0,1.0 +dense+rerank,343,10,1219.21533,0.57064,1.0,0.33333,1.0 +dense+rerank,350,10,1095.67363,1.0,1.0,1.0,1.0 +dense+rerank,354,10,1141.55521,1.0,1.0,1.0,1.0 +dense+rerank,362,10,1201.13904,1.0,1.0,1.0,1.0 +dense+rerank,380,10,1157.986,1.0,1.0,1.0,1.0 +dense+rerank,384,10,1209.70408,0.0,0.0,0.0,0.0 +dense+rerank,385,10,1129.46833,0.69343,1.0,0.5,1.0 +dense+rerank,386,10,1010.65254,1.0,1.0,1.0,1.0 +dense+rerank,388,10,1031.92321,0.5,1.0,0.33333,1.0 +dense+rerank,399,10,1226.45392,1.0,1.0,1.0,1.0 +dense+rerank,410,10,1218.72167,1.0,1.0,1.0,1.0 +dense+rerank,411,10,1151.06154,1.0,1.0,1.0,1.0 +dense+rerank,415,10,1194.30833,0.5,1.0,0.33333,1.0 +dense+rerank,421,10,1166.98192,0.0,0.0,0.0,0.0 +dense+rerank,431,10,1160.33221,0.30103,1.0,0.11111,0.0 +dense+rerank,436,10,644.32983,1.0,1.0,1.0,1.0 +dense+rerank,437,10,1213.83483,0.0,0.0,0.0,0.0 +dense+rerank,439,10,1153.60346,1.0,1.0,1.0,1.0 +dense+rerank,440,10,1207.20413,1.0,1.0,1.0,1.0 +dense+rerank,443,10,1199.29738,1.0,1.0,1.0,1.0 +dense+rerank,452,10,1272.42088,0.2372,0.5,0.2,0.0 +dense+rerank,475,10,1241.76658,0.38685,1.0,0.2,0.0 +dense+rerank,478,10,1166.55088,1.0,1.0,1.0,1.0 +dense+rerank,491,10,1204.89571,1.0,1.0,1.0,1.0 +dense+rerank,501,10,1231.85229,1.0,1.0,1.0,1.0 +dense+rerank,502,10,1106.589,0.0,0.0,0.0,0.0 +dense+rerank,507,10,1234.72412,0.63093,1.0,0.5,1.0 +dense+rerank,508,10,1177.19121,0.0,0.0,0.0,0.0 +dense+rerank,513,10,1273.34129,0.63093,1.0,0.5,1.0 +dense+rerank,514,10,1215.58454,0.5,1.0,0.33333,1.0 +dense+rerank,516,10,1221.75671,1.0,1.0,1.0,1.0 +dense+rerank,517,10,1287.08496,0.0,0.0,0.0,0.0 +dense+rerank,521,10,1222.85725,1.0,1.0,1.0,1.0 +dense+rerank,525,10,1225.68242,1.0,1.0,1.0,1.0 +dense+rerank,527,10,1276.91967,0.35621,1.0,0.16667,0.0 +dense+rerank,528,10,1200.24383,0.5,1.0,0.33333,1.0 +dense+rerank,532,10,1017.73696,1.0,1.0,1.0,1.0 +dense+rerank,533,10,1269.33033,1.0,1.0,1.0,1.0 +dense+rerank,535,10,1267.31738,0.0,0.0,0.0,0.0 +dense+rerank,536,10,1223.38237,1.0,1.0,1.0,1.0 +dense+rerank,539,10,1180.60712,1.0,1.0,1.0,1.0 +dense+rerank,540,10,1236.61354,0.30657,0.5,0.33333,1.0 +dense+rerank,544,10,1272.20812,0.38685,1.0,0.2,0.0 +dense+rerank,549,10,1162.25979,1.0,1.0,1.0,1.0 +dense+rerank,551,10,1134.31863,0.5,1.0,0.33333,1.0 +dense+rerank,552,10,1254.42571,1.0,1.0,1.0,1.0 +dense+rerank,554,10,1287.709,0.35621,1.0,0.16667,0.0 +dense+rerank,560,10,1016.27913,0.0,0.0,0.0,0.0 +dense+rerank,569,10,1253.39846,1.0,1.0,1.0,1.0 +dense+rerank,575,10,1094.32921,1.0,1.0,1.0,1.0 +dense+rerank,577,10,1237.64658,0.0,0.0,0.0,0.0 +dense+rerank,578,10,1283.61446,1.0,1.0,1.0,1.0 +dense+rerank,587,10,1228.42983,0.0,0.0,0.0,0.0 +dense+rerank,589,10,1264.34492,1.0,1.0,1.0,1.0 +dense+rerank,593,10,1313.3925,1.0,1.0,1.0,1.0 +dense+rerank,597,10,1317.65088,0.88546,1.0,1.0,1.0 +dense+rerank,598,10,1255.28896,0.33333,1.0,0.14286,0.0 +dense+rerank,613,10,1222.13438,1.0,1.0,1.0,1.0 +dense+rerank,619,10,1177.49212,0.61315,0.5,1.0,1.0 +dense+rerank,623,10,1231.70329,0.0,0.0,0.0,0.0 +dense+rerank,628,10,1221.88025,1.0,1.0,1.0,1.0 +dense+rerank,636,10,1256.3535,1.0,1.0,1.0,1.0 +dense+rerank,637,10,1306.50225,0.63093,1.0,0.5,1.0 +dense+rerank,641,10,1207.12729,0.79772,1.0,1.0,1.0 +dense+rerank,644,10,1024.95317,1.0,1.0,1.0,1.0 +dense+rerank,649,10,1278.44833,1.0,1.0,1.0,1.0 +dense+rerank,659,10,1224.42321,0.43068,1.0,0.25,0.0 +dense+rerank,660,10,1018.12954,0.0,0.0,0.0,0.0 +dense+rerank,674,10,1241.49804,1.0,1.0,1.0,1.0 +dense+rerank,684,10,1332.01888,0.43068,1.0,0.25,0.0 +dense+rerank,690,10,1267.83592,0.0,0.0,0.0,0.0 +dense+rerank,691,10,1263.50125,0.63093,1.0,0.5,1.0 +dense+rerank,692,10,1254.20613,1.0,1.0,1.0,1.0 +dense+rerank,693,10,1019.27167,1.0,1.0,1.0,1.0 +dense+rerank,700,10,1338.41037,0.33333,1.0,0.14286,0.0 +dense+rerank,702,10,1304.55204,0.31546,1.0,0.125,0.0 +dense+rerank,715,10,1210.27488,0.0,0.0,0.0,0.0 +dense+rerank,716,10,1289.25975,0.0,0.0,0.0,0.0 +dense+rerank,718,10,1318.40967,1.0,1.0,1.0,1.0 +dense+rerank,721,10,1164.36604,1.0,1.0,1.0,1.0 +dense+rerank,723,10,1206.86621,1.0,1.0,1.0,1.0 +dense+rerank,727,10,1354.23867,0.33333,1.0,0.14286,0.0 +dense+rerank,728,10,1295.01008,0.81753,1.0,1.0,1.0 +dense+rerank,729,10,1306.22275,1.0,1.0,1.0,1.0 +dense+rerank,742,10,1259.23683,1.0,1.0,1.0,1.0 +dense+rerank,743,10,1320.77721,1.0,1.0,1.0,1.0 +dense+rerank,744,10,1289.94204,0.63093,1.0,0.5,1.0 +dense+rerank,756,10,782.90746,0.43068,1.0,0.25,0.0 +dense+rerank,759,10,1147.06842,1.0,1.0,1.0,1.0 +dense+rerank,768,10,1356.92212,0.0,0.0,0.0,0.0 +dense+rerank,770,10,1346.4825,1.0,1.0,1.0,1.0 +dense+rerank,775,10,1091.58238,0.33333,1.0,0.14286,0.0 +dense+rerank,781,10,1204.31346,0.63093,1.0,0.5,1.0 +dense+rerank,783,10,1315.47383,0.31546,1.0,0.125,0.0 +dense+rerank,784,10,1367.59442,1.0,1.0,1.0,1.0 +dense+rerank,785,10,1271.25579,0.0,0.0,0.0,0.0 +dense+rerank,793,10,1374.2505,1.0,1.0,1.0,1.0 +dense+rerank,800,10,832.09312,0.63093,1.0,0.5,1.0 +dense+rerank,805,10,1367.24983,1.0,1.0,1.0,1.0 +dense+rerank,808,10,1035.65804,1.0,1.0,1.0,1.0 +dense+rerank,811,10,1331.49708,1.0,1.0,1.0,1.0 +dense+rerank,814,10,1298.90671,1.0,1.0,1.0,1.0 +dense+rerank,820,10,734.71379,0.0,0.0,0.0,0.0 +dense+rerank,821,10,715.60158,0.0,0.0,0.0,0.0 +dense+rerank,823,10,1386.79375,1.0,1.0,1.0,1.0 +dense+rerank,830,10,1280.23071,0.0,0.0,0.0,0.0 +dense+rerank,831,10,1211.38642,0.0,0.0,0.0,0.0 +dense+rerank,832,10,1099.81162,1.0,1.0,1.0,1.0 +dense+rerank,834,10,1390.78821,0.0,0.0,0.0,0.0 +dense+rerank,837,10,1286.14737,0.63093,1.0,0.5,1.0 +dense+rerank,839,10,1330.73671,1.0,1.0,1.0,1.0 +dense+rerank,845,10,1158.11633,1.0,1.0,1.0,1.0 +dense+rerank,847,10,1314.93787,1.0,1.0,1.0,1.0 +dense+rerank,852,10,1391.32725,0.63093,1.0,0.5,1.0 +dense+rerank,859,10,1309.40925,0.63093,1.0,0.5,1.0 +dense+rerank,870,10,1378.72292,0.0,0.0,0.0,0.0 +dense+rerank,873,10,1277.28154,0.41369,0.6,0.33333,1.0 +dense+rerank,879,10,1035.25763,1.0,1.0,1.0,1.0 +dense+rerank,880,10,1252.70358,1.0,1.0,1.0,1.0 +dense+rerank,882,10,1302.36271,1.0,1.0,1.0,1.0 +dense+rerank,887,10,1241.95175,0.0,0.0,0.0,0.0 +dense+rerank,903,10,1153.9285,1.0,1.0,1.0,1.0 +dense+rerank,904,10,1437.78129,1.0,1.0,1.0,1.0 +dense+rerank,907,10,1336.22767,1.0,1.0,1.0,1.0 +dense+rerank,911,10,1318.16671,1.0,1.0,1.0,1.0 +dense+rerank,913,10,1368.51012,0.0,0.0,0.0,0.0 +dense+rerank,914,10,1288.07604,0.0,0.0,0.0,0.0 +dense+rerank,921,10,1416.08108,1.0,1.0,1.0,1.0 +dense+rerank,922,10,1367.46408,1.0,1.0,1.0,1.0 +dense+rerank,936,10,1291.72254,1.0,1.0,1.0,1.0 +dense+rerank,956,10,1191.21333,1.0,1.0,1.0,1.0 +dense+rerank,957,10,782.3285,1.0,1.0,1.0,1.0 +dense+rerank,960,10,1042.81196,1.0,1.0,1.0,1.0 +dense+rerank,967,10,1378.92479,0.91972,1.0,1.0,1.0 +dense+rerank,971,10,1265.56425,0.95583,1.0,1.0,1.0 +dense+rerank,975,10,1356.11642,0.0,0.0,0.0,0.0 +dense+rerank,982,10,1223.87158,1.0,1.0,1.0,1.0 +dense+rerank,985,10,837.42871,1.0,1.0,1.0,1.0 +dense+rerank,993,10,1326.82354,1.0,1.0,1.0,1.0 +dense+rerank,1012,10,1300.78913,1.0,1.0,1.0,1.0 +dense+rerank,1014,10,1264.2005,1.0,1.0,1.0,1.0 +dense+rerank,1019,10,1335.16812,0.63093,1.0,0.5,1.0 +dense+rerank,1020,10,1392.76958,0.63093,1.0,0.5,1.0 +dense+rerank,1021,10,1523.63433,0.63093,1.0,0.5,1.0 +dense+rerank,1024,10,1171.01167,0.0,0.0,0.0,0.0 +dense+rerank,1029,10,1142.28096,0.76536,0.66667,1.0,1.0 +dense+rerank,1041,10,1480.74013,0.21841,0.5,0.16667,0.0 +dense+rerank,1049,10,1522.01663,0.43068,1.0,0.25,0.0 +dense+rerank,1062,10,1294.23071,1.0,1.0,1.0,1.0 +dense+rerank,1086,10,1186.11058,1.0,1.0,1.0,1.0 +dense+rerank,1088,10,1434.64033,1.0,1.0,1.0,1.0 +dense+rerank,1089,10,867.73158,1.0,1.0,1.0,1.0 +dense+rerank,1099,10,1204.57187,0.43068,1.0,0.25,0.0 +dense+rerank,1100,10,1400.83846,0.43068,1.0,0.25,0.0 +dense+rerank,1104,10,1383.66146,1.0,1.0,1.0,1.0 +dense+rerank,1107,10,1290.18037,1.0,1.0,1.0,1.0 +dense+rerank,1110,10,1014.35742,0.0,0.0,0.0,0.0 +dense+rerank,1121,10,1355.14646,1.0,1.0,1.0,1.0 +dense+rerank,1130,10,1263.18667,1.0,1.0,1.0,1.0 +dense+rerank,1132,10,1152.45962,0.61315,0.5,1.0,1.0 +dense+rerank,1137,10,1472.17875,1.0,1.0,1.0,1.0 +dense+rerank,1140,10,1464.82358,1.0,1.0,1.0,1.0 +dense+rerank,1144,10,1014.74312,1.0,1.0,1.0,1.0 +dense+rerank,1146,10,1429.43908,1.0,1.0,1.0,1.0 +dense+rerank,1150,10,1039.84946,1.0,1.0,1.0,1.0 +dense+rerank,1163,10,1421.78921,1.0,1.0,1.0,1.0 +dense+rerank,1175,10,1304.94808,0.0,0.0,0.0,0.0 +dense+rerank,1179,10,1346.11379,0.0,0.0,0.0,0.0 +dense+rerank,1180,10,1270.70462,1.0,1.0,1.0,1.0 +dense+rerank,1185,10,1378.10992,1.0,1.0,1.0,1.0 +dense+rerank,1187,10,850.40967,1.0,1.0,1.0,1.0 +dense+rerank,1191,10,802.57583,0.38685,1.0,0.2,0.0 +dense+rerank,1194,10,792.37929,1.0,1.0,1.0,1.0 +dense+rerank,1196,10,1441.02879,0.43068,1.0,0.25,0.0 +dense+rerank,1197,10,1382.16538,0.43068,1.0,0.25,0.0 +dense+rerank,1199,10,1355.71854,0.0,0.0,0.0,0.0 +dense+rerank,1200,10,1488.09796,0.0,0.0,0.0,0.0 +dense+rerank,1202,10,1432.58658,0.63093,1.0,0.5,1.0 +dense+rerank,1204,10,1341.20633,0.43068,1.0,0.25,0.0 +dense+rerank,1207,10,1410.52737,1.0,1.0,1.0,1.0 +dense+rerank,1213,10,1440.18763,0.0,0.0,0.0,0.0 +dense+rerank,1216,10,1339.29817,1.0,1.0,1.0,1.0 +dense+rerank,1221,10,1490.39733,0.38685,1.0,0.2,0.0 +dense+rerank,1225,10,1298.27137,1.0,1.0,1.0,1.0 +dense+rerank,1226,10,1503.97896,0.0,0.0,0.0,0.0 +dense+rerank,1232,10,1338.69542,1.0,1.0,1.0,1.0 +dense+rerank,1241,10,1265.85608,1.0,1.0,1.0,1.0 +dense+rerank,1245,10,1511.13012,0.0,0.0,0.0,0.0 +dense+rerank,1259,10,1402.32779,1.0,1.0,1.0,1.0 +dense+rerank,1262,10,1422.40629,1.0,1.0,1.0,1.0 +dense+rerank,1266,10,1276.94217,0.5,1.0,0.33333,1.0 +dense+rerank,1270,10,1364.24058,1.0,1.0,1.0,1.0 +dense+rerank,1271,10,1552.55583,1.0,1.0,1.0,1.0 +dense+rerank,1272,10,1565.85658,0.5,1.0,0.33333,1.0 +dense+rerank,1273,10,1398.25283,1.0,1.0,1.0,1.0 +dense+rerank,1274,10,1435.916,0.93252,1.0,1.0,1.0 +dense+rerank,1278,10,1311.95946,0.31546,1.0,0.125,0.0 +dense+rerank,1279,10,1261.17108,0.35621,1.0,0.16667,0.0 +dense+rerank,1280,10,1384.3155,0.35621,1.0,0.16667,0.0 +dense+rerank,1281,10,1509.63433,0.31546,1.0,0.125,0.0 +dense+rerank,1282,10,1317.51742,1.0,1.0,1.0,1.0 +dense+rerank,1290,10,1405.91438,1.0,1.0,1.0,1.0 +dense+rerank,1292,10,1510.95004,0.5,1.0,0.33333,1.0 +dense+rerank,1298,10,1497.57583,1.0,1.0,1.0,1.0 +dense+rerank,1303,10,1517.92283,1.0,1.0,1.0,1.0 +dense+rerank,1316,10,1493.45721,0.0,0.0,0.0,0.0 +dense+rerank,1319,10,1508.16358,1.0,1.0,1.0,1.0 +dense+rerank,1320,10,1404.60271,1.0,1.0,1.0,1.0 +dense+rerank,1332,10,1247.21688,0.0,0.0,0.0,0.0 +dense+rerank,1335,10,1438.0875,1.0,1.0,1.0,1.0 +dense+rerank,1336,10,1407.30042,1.0,1.0,1.0,1.0 +dense+rerank,1337,10,1448.51329,0.63093,1.0,0.5,1.0 +dense+rerank,1339,10,1440.82037,1.0,1.0,1.0,1.0 +dense+rerank,1344,10,1354.58608,0.5,1.0,0.33333,1.0 +dense+rerank,1352,10,1496.97158,1.0,1.0,1.0,1.0 +dense+rerank,1359,10,1601.553,1.0,1.0,1.0,1.0 +dense+rerank,1362,10,1339.81054,0.0,0.0,0.0,0.0 +dense+rerank,1363,10,1347.85971,0.0,0.0,0.0,0.0 +dense+rerank,1368,10,1549.46563,1.0,1.0,1.0,1.0 +dense+rerank,1370,10,1530.59171,0.63093,1.0,0.5,1.0 +dense+rerank,1379,10,1161.82746,0.83268,1.0,1.0,1.0 +dense+rerank,1382,10,1302.49587,1.0,1.0,1.0,1.0 +dense+rerank,1385,10,1345.51508,1.0,1.0,1.0,1.0 +dense+rerank,1389,10,1397.0295,1.0,1.0,1.0,1.0 +dense+rerank,1395,10,1520.75004,1.0,1.0,1.0,1.0 +hybrid+rerank,1,10,2090.27013,0.0,0.0,0.0,0.0 +hybrid+rerank,3,10,2010.44121,1.0,1.0,1.0,1.0 +hybrid+rerank,5,10,2408.31529,1.0,1.0,1.0,1.0 +hybrid+rerank,13,10,2227.87754,0.0,0.0,0.0,0.0 +hybrid+rerank,36,10,2136.61042,0.38896,1.0,0.14286,0.0 +hybrid+rerank,42,10,2332.12125,1.0,1.0,1.0,1.0 +hybrid+rerank,48,10,2510.38933,0.5,1.0,0.33333,1.0 +hybrid+rerank,49,10,1940.52475,1.0,1.0,1.0,1.0 +hybrid+rerank,50,10,1684.68137,1.0,1.0,1.0,1.0 +hybrid+rerank,51,10,2017.96217,0.63093,1.0,0.5,1.0 +hybrid+rerank,53,10,1849.28279,1.0,1.0,1.0,1.0 +hybrid+rerank,54,10,1667.66521,0.30103,1.0,0.11111,0.0 +hybrid+rerank,56,10,1979.10104,1.0,1.0,1.0,1.0 +hybrid+rerank,57,10,1893.61329,1.0,1.0,1.0,1.0 +hybrid+rerank,70,10,1712.0015,0.0,0.0,0.0,0.0 +hybrid+rerank,72,10,2110.36075,1.0,1.0,1.0,1.0 +hybrid+rerank,75,10,2019.49208,1.0,1.0,1.0,1.0 +hybrid+rerank,94,10,1648.26275,0.0,0.0,0.0,0.0 +hybrid+rerank,99,10,2204.94579,1.0,1.0,1.0,1.0 +hybrid+rerank,100,10,2056.77608,1.0,1.0,1.0,1.0 +hybrid+rerank,113,10,1612.507,1.0,1.0,1.0,1.0 +hybrid+rerank,115,10,1802.73954,1.0,1.0,1.0,1.0 +hybrid+rerank,118,10,1775.52592,1.0,1.0,1.0,1.0 +hybrid+rerank,124,10,1765.43708,1.0,1.0,1.0,1.0 +hybrid+rerank,127,10,2291.438,0.63093,1.0,0.5,1.0 +hybrid+rerank,128,10,1951.41054,0.0,0.0,0.0,0.0 +hybrid+rerank,129,10,2436.76763,1.0,1.0,1.0,1.0 +hybrid+rerank,130,10,1979.64983,1.0,1.0,1.0,1.0 +hybrid+rerank,132,10,2032.37179,0.0,0.0,0.0,0.0 +hybrid+rerank,133,10,2135.88304,0.51477,0.6,0.5,1.0 +hybrid+rerank,137,10,1608.78846,1.0,1.0,1.0,1.0 +hybrid+rerank,141,10,1722.16108,0.91972,1.0,1.0,1.0 +hybrid+rerank,142,10,1981.51921,1.0,1.0,1.0,1.0 +hybrid+rerank,143,10,1985.00258,1.0,1.0,1.0,1.0 +hybrid+rerank,146,10,1921.43592,1.0,1.0,1.0,1.0 +hybrid+rerank,148,10,1665.05342,1.0,1.0,1.0,1.0 +hybrid+rerank,163,10,1869.87633,0.63093,1.0,0.5,1.0 +hybrid+rerank,171,10,2008.43421,1.0,1.0,1.0,1.0 +hybrid+rerank,179,10,1801.68638,0.90522,1.0,1.0,1.0 +hybrid+rerank,180,10,2140.18867,1.0,1.0,1.0,1.0 +hybrid+rerank,183,10,1833.19154,1.0,1.0,1.0,1.0 +hybrid+rerank,185,10,1907.92658,0.43068,1.0,0.25,0.0 +hybrid+rerank,198,10,1670.9625,0.0,0.0,0.0,0.0 +hybrid+rerank,208,10,1975.89458,1.0,1.0,1.0,1.0 +hybrid+rerank,212,10,1777.58537,0.5,1.0,0.33333,1.0 +hybrid+rerank,213,10,2519.92317,0.0,0.0,0.0,0.0 +hybrid+rerank,216,10,2067.53579,1.0,1.0,1.0,1.0 +hybrid+rerank,217,10,1713.29779,1.0,1.0,1.0,1.0 +hybrid+rerank,218,10,2177.98033,1.0,1.0,1.0,1.0 +hybrid+rerank,219,10,1646.415,1.0,1.0,1.0,1.0 +hybrid+rerank,230,10,2037.36117,1.0,1.0,1.0,1.0 +hybrid+rerank,232,10,1549.17733,0.63093,1.0,0.5,1.0 +hybrid+rerank,233,10,1749.50217,1.0,1.0,1.0,1.0 +hybrid+rerank,236,10,1927.60525,1.0,1.0,1.0,1.0 +hybrid+rerank,237,10,2002.33171,0.43068,1.0,0.25,0.0 +hybrid+rerank,238,10,2044.15904,0.0,0.0,0.0,0.0 +hybrid+rerank,239,10,1906.41612,0.0,0.0,0.0,0.0 +hybrid+rerank,248,10,1735.49621,1.0,1.0,1.0,1.0 +hybrid+rerank,249,10,2072.76825,1.0,1.0,1.0,1.0 +hybrid+rerank,261,10,1579.78292,0.38685,0.5,0.5,1.0 +hybrid+rerank,268,10,1869.72333,0.63093,1.0,0.5,1.0 +hybrid+rerank,269,10,1846.1585,0.63093,1.0,0.5,1.0 +hybrid+rerank,274,10,2519.90437,1.0,1.0,1.0,1.0 +hybrid+rerank,275,10,1909.52717,0.87108,1.0,1.0,1.0 +hybrid+rerank,279,10,2075.65521,1.0,1.0,1.0,1.0 +hybrid+rerank,294,10,1984.4265,1.0,1.0,1.0,1.0 +hybrid+rerank,295,10,2007.31529,1.0,1.0,1.0,1.0 +hybrid+rerank,298,10,2061.11587,1.0,1.0,1.0,1.0 +hybrid+rerank,300,10,2045.63888,0.5,1.0,0.33333,1.0 +hybrid+rerank,303,10,1893.98512,0.0,0.0,0.0,0.0 +hybrid+rerank,312,10,1918.0075,0.0,0.0,0.0,0.0 +hybrid+rerank,314,10,2088.01942,0.0,0.0,0.0,0.0 +hybrid+rerank,324,10,2131.2455,0.30103,1.0,0.11111,0.0 +hybrid+rerank,327,10,2050.138,1.0,1.0,1.0,1.0 +hybrid+rerank,338,10,2726.83662,1.0,1.0,1.0,1.0 +hybrid+rerank,343,10,2563.9085,0.57064,1.0,0.33333,1.0 +hybrid+rerank,350,10,2038.675,1.0,1.0,1.0,1.0 +hybrid+rerank,354,10,2036.98754,1.0,1.0,1.0,1.0 +hybrid+rerank,362,10,1913.41625,1.0,1.0,1.0,1.0 +hybrid+rerank,380,10,1541.48213,1.0,1.0,1.0,1.0 +hybrid+rerank,384,10,1645.71412,0.0,0.0,0.0,0.0 +hybrid+rerank,385,10,1914.24683,0.65092,1.0,0.5,1.0 +hybrid+rerank,386,10,1955.12871,1.0,1.0,1.0,1.0 +hybrid+rerank,388,10,1983.46933,0.30103,1.0,0.11111,0.0 +hybrid+rerank,399,10,1719.79825,1.0,1.0,1.0,1.0 +hybrid+rerank,410,10,1403.68129,1.0,1.0,1.0,1.0 +hybrid+rerank,411,10,1646.98337,1.0,1.0,1.0,1.0 +hybrid+rerank,415,10,1625.32275,0.5,1.0,0.33333,1.0 +hybrid+rerank,421,10,1820.59046,0.0,0.0,0.0,0.0 +hybrid+rerank,431,10,1641.22817,0.0,0.0,0.0,0.0 +hybrid+rerank,436,10,1735.14175,1.0,1.0,1.0,1.0 +hybrid+rerank,437,10,1768.60996,0.0,0.0,0.0,0.0 +hybrid+rerank,439,10,1737.80158,1.0,1.0,1.0,1.0 +hybrid+rerank,440,10,1615.71771,1.0,1.0,1.0,1.0 +hybrid+rerank,443,10,1660.83675,1.0,1.0,1.0,1.0 +hybrid+rerank,452,10,1734.74379,0.60526,1.0,0.5,1.0 +hybrid+rerank,475,10,1651.56329,0.38685,1.0,0.2,0.0 +hybrid+rerank,478,10,1701.759,1.0,1.0,1.0,1.0 +hybrid+rerank,491,10,1858.05954,1.0,1.0,1.0,1.0 +hybrid+rerank,501,10,1505.67358,1.0,1.0,1.0,1.0 +hybrid+rerank,502,10,1905.43829,0.0,0.0,0.0,0.0 +hybrid+rerank,507,10,1616.59108,0.63093,1.0,0.5,1.0 +hybrid+rerank,508,10,1573.53746,0.0,0.0,0.0,0.0 +hybrid+rerank,513,10,1864.44138,0.63093,1.0,0.5,1.0 +hybrid+rerank,514,10,1723.42604,0.5,1.0,0.33333,1.0 +hybrid+rerank,516,10,1363.56175,1.0,1.0,1.0,1.0 +hybrid+rerank,517,10,1946.12846,0.0,0.0,0.0,0.0 +hybrid+rerank,521,10,2027.91887,1.0,1.0,1.0,1.0 +hybrid+rerank,525,10,1613.39079,1.0,1.0,1.0,1.0 +hybrid+rerank,527,10,1777.60146,0.0,0.0,0.0,0.0 +hybrid+rerank,528,10,1888.90408,0.5,1.0,0.33333,1.0 +hybrid+rerank,532,10,2171.88321,1.0,1.0,1.0,1.0 +hybrid+rerank,533,10,1874.37408,1.0,1.0,1.0,1.0 +hybrid+rerank,535,10,1694.07637,0.0,0.0,0.0,0.0 +hybrid+rerank,536,10,1957.14954,1.0,1.0,1.0,1.0 +hybrid+rerank,539,10,1667.14725,1.0,1.0,1.0,1.0 +hybrid+rerank,540,10,1521.31008,0.30657,0.5,0.33333,1.0 +hybrid+rerank,544,10,1730.57583,0.38685,1.0,0.2,0.0 +hybrid+rerank,549,10,1675.22567,1.0,1.0,1.0,1.0 +hybrid+rerank,551,10,1485.90633,0.5,1.0,0.33333,1.0 +hybrid+rerank,552,10,1620.34275,1.0,1.0,1.0,1.0 +hybrid+rerank,554,10,1646.75838,0.30103,1.0,0.11111,0.0 +hybrid+rerank,560,10,1608.01358,0.0,0.0,0.0,0.0 +hybrid+rerank,569,10,1415.37083,1.0,1.0,1.0,1.0 +hybrid+rerank,575,10,1871.85413,1.0,1.0,1.0,1.0 +hybrid+rerank,577,10,1856.17796,0.0,0.0,0.0,0.0 +hybrid+rerank,578,10,1792.79525,1.0,1.0,1.0,1.0 +hybrid+rerank,587,10,1748.41221,0.63093,1.0,0.5,1.0 +hybrid+rerank,589,10,1926.57525,1.0,1.0,1.0,1.0 +hybrid+rerank,593,10,1954.00621,1.0,1.0,1.0,1.0 +hybrid+rerank,597,10,1640.59283,0.88546,1.0,1.0,1.0 +hybrid+rerank,598,10,1496.493,0.33333,1.0,0.14286,0.0 +hybrid+rerank,613,10,1748.33842,1.0,1.0,1.0,1.0 +hybrid+rerank,619,10,1879.96133,0.38685,0.5,0.5,1.0 +hybrid+rerank,623,10,1880.93779,0.0,0.0,0.0,0.0 +hybrid+rerank,628,10,1906.13812,1.0,1.0,1.0,1.0 +hybrid+rerank,636,10,1807.186,1.0,1.0,1.0,1.0 +hybrid+rerank,637,10,1828.02658,0.63093,1.0,0.5,1.0 +hybrid+rerank,641,10,1845.38108,0.79772,1.0,1.0,1.0 +hybrid+rerank,644,10,1702.25558,1.0,1.0,1.0,1.0 +hybrid+rerank,649,10,1991.12733,1.0,1.0,1.0,1.0 +hybrid+rerank,659,10,1681.25429,0.35621,1.0,0.16667,0.0 +hybrid+rerank,660,10,2132.37383,0.0,0.0,0.0,0.0 +hybrid+rerank,674,10,1766.96696,1.0,1.0,1.0,1.0 +hybrid+rerank,684,10,1673.49567,0.43068,1.0,0.25,0.0 +hybrid+rerank,690,10,2042.92654,1.0,1.0,1.0,1.0 +hybrid+rerank,691,10,1923.85317,0.63093,1.0,0.5,1.0 +hybrid+rerank,692,10,1855.48096,1.0,1.0,1.0,1.0 +hybrid+rerank,693,10,2001.52625,1.0,1.0,1.0,1.0 +hybrid+rerank,700,10,1879.43017,0.33333,1.0,0.14286,0.0 +hybrid+rerank,702,10,1830.93671,0.31546,1.0,0.125,0.0 +hybrid+rerank,715,10,1984.67242,0.33333,1.0,0.14286,0.0 +hybrid+rerank,716,10,1843.64688,0.0,0.0,0.0,0.0 +hybrid+rerank,718,10,2130.37004,0.63093,1.0,0.5,1.0 +hybrid+rerank,721,10,1930.56496,1.0,1.0,1.0,1.0 +hybrid+rerank,723,10,2040.30267,1.0,1.0,1.0,1.0 +hybrid+rerank,727,10,1831.10775,0.31546,1.0,0.125,0.0 +hybrid+rerank,728,10,1771.497,0.80657,1.0,1.0,1.0 +hybrid+rerank,729,10,2075.54033,1.0,1.0,1.0,1.0 +hybrid+rerank,742,10,2176.91442,1.0,1.0,1.0,1.0 +hybrid+rerank,743,10,2233.65008,1.0,1.0,1.0,1.0 +hybrid+rerank,744,10,1977.66183,0.63093,1.0,0.5,1.0 +hybrid+rerank,756,10,1701.31379,0.43068,1.0,0.25,0.0 +hybrid+rerank,759,10,1632.10454,1.0,1.0,1.0,1.0 +hybrid+rerank,768,10,2073.28892,0.0,0.0,0.0,0.0 +hybrid+rerank,770,10,2359.30912,1.0,1.0,1.0,1.0 +hybrid+rerank,775,10,1748.27492,0.33333,1.0,0.14286,0.0 +hybrid+rerank,781,10,2052.22308,0.63093,1.0,0.5,1.0 +hybrid+rerank,783,10,2025.13821,0.28906,1.0,0.1,0.0 +hybrid+rerank,784,10,2053.49696,1.0,1.0,1.0,1.0 +hybrid+rerank,785,10,1901.22742,0.28906,1.0,0.1,0.0 +hybrid+rerank,793,10,1947.63167,1.0,1.0,1.0,1.0 +hybrid+rerank,800,10,1952.64079,0.63093,1.0,0.5,1.0 +hybrid+rerank,805,10,2006.04583,1.0,1.0,1.0,1.0 +hybrid+rerank,808,10,1955.03763,1.0,1.0,1.0,1.0 +hybrid+rerank,811,10,1857.52904,1.0,1.0,1.0,1.0 +hybrid+rerank,814,10,2230.52508,1.0,1.0,1.0,1.0 +hybrid+rerank,820,10,2076.40117,0.35621,1.0,0.16667,0.0 +hybrid+rerank,821,10,1927.13742,0.38685,1.0,0.2,0.0 +hybrid+rerank,823,10,2076.31371,1.0,1.0,1.0,1.0 +hybrid+rerank,830,10,2069.50717,0.0,0.0,0.0,0.0 +hybrid+rerank,831,10,2257.04883,0.0,0.0,0.0,0.0 +hybrid+rerank,832,10,2058.49158,1.0,1.0,1.0,1.0 +hybrid+rerank,834,10,2135.10392,0.38685,1.0,0.2,0.0 +hybrid+rerank,837,10,1985.80842,0.63093,1.0,0.5,1.0 +hybrid+rerank,839,10,1830.47492,1.0,1.0,1.0,1.0 +hybrid+rerank,845,10,1668.53771,1.0,1.0,1.0,1.0 +hybrid+rerank,847,10,1713.83288,1.0,1.0,1.0,1.0 +hybrid+rerank,852,10,2161.18183,0.63093,1.0,0.5,1.0 +hybrid+rerank,859,10,2169.11758,0.63093,1.0,0.5,1.0 +hybrid+rerank,870,10,1790.34392,0.0,0.0,0.0,0.0 +hybrid+rerank,873,10,2033.87179,0.25202,0.4,0.2,0.0 +hybrid+rerank,879,10,2323.33313,1.0,1.0,1.0,1.0 +hybrid+rerank,880,10,2388.20787,1.0,1.0,1.0,1.0 +hybrid+rerank,882,10,1964.70054,1.0,1.0,1.0,1.0 +hybrid+rerank,887,10,2043.975,0.0,0.0,0.0,0.0 +hybrid+rerank,903,10,1975.62554,1.0,1.0,1.0,1.0 +hybrid+rerank,904,10,1791.522,1.0,1.0,1.0,1.0 +hybrid+rerank,907,10,1987.95929,1.0,1.0,1.0,1.0 +hybrid+rerank,911,10,2137.05721,1.0,1.0,1.0,1.0 +hybrid+rerank,913,10,2253.78683,0.0,0.0,0.0,0.0 +hybrid+rerank,914,10,2184.30963,0.0,0.0,0.0,0.0 +hybrid+rerank,921,10,2007.84025,1.0,1.0,1.0,1.0 +hybrid+rerank,922,10,1925.63483,1.0,1.0,1.0,1.0 +hybrid+rerank,936,10,2277.14917,1.0,1.0,1.0,1.0 +hybrid+rerank,956,10,2257.80517,1.0,1.0,1.0,1.0 +hybrid+rerank,957,10,1293.80929,1.0,1.0,1.0,1.0 +hybrid+rerank,960,10,2246.12925,1.0,1.0,1.0,1.0 +hybrid+rerank,967,10,1707.28317,0.91972,1.0,1.0,1.0 +hybrid+rerank,971,10,1523.04808,0.95583,1.0,1.0,1.0 +hybrid+rerank,975,10,1937.28458,0.0,0.0,0.0,0.0 +hybrid+rerank,982,10,1867.97454,1.0,1.0,1.0,1.0 +hybrid+rerank,985,10,2098.27108,1.0,1.0,1.0,1.0 +hybrid+rerank,993,10,1893.69758,1.0,1.0,1.0,1.0 +hybrid+rerank,1012,10,1922.34408,1.0,1.0,1.0,1.0 +hybrid+rerank,1014,10,2047.818,1.0,1.0,1.0,1.0 +hybrid+rerank,1019,10,1732.17921,0.63093,1.0,0.5,1.0 +hybrid+rerank,1020,10,1737.86654,0.63093,1.0,0.5,1.0 +hybrid+rerank,1021,10,1675.41483,0.63093,1.0,0.5,1.0 +hybrid+rerank,1024,10,1871.39908,1.0,1.0,1.0,1.0 +hybrid+rerank,1029,10,1590.53333,0.76536,0.66667,1.0,1.0 +hybrid+rerank,1041,10,1343.32067,0.21841,0.5,0.16667,0.0 +hybrid+rerank,1049,10,2000.37804,0.43068,1.0,0.25,0.0 +hybrid+rerank,1062,10,1521.338,1.0,1.0,1.0,1.0 +hybrid+rerank,1086,10,2364.48154,1.0,1.0,1.0,1.0 +hybrid+rerank,1088,10,1731.91571,1.0,1.0,1.0,1.0 +hybrid+rerank,1089,10,1533.21892,1.0,1.0,1.0,1.0 +hybrid+rerank,1099,10,1678.943,0.43068,1.0,0.25,0.0 +hybrid+rerank,1100,10,1625.54188,0.43068,1.0,0.25,0.0 +hybrid+rerank,1104,10,1976.7515,1.0,1.0,1.0,1.0 +hybrid+rerank,1107,10,1629.14042,1.0,1.0,1.0,1.0 +hybrid+rerank,1110,10,1981.11683,0.0,0.0,0.0,0.0 +hybrid+rerank,1121,10,1543.38996,1.0,1.0,1.0,1.0 +hybrid+rerank,1130,10,1735.64625,1.0,1.0,1.0,1.0 +hybrid+rerank,1132,10,1691.36554,0.61315,0.5,1.0,1.0 +hybrid+rerank,1137,10,1890.60912,1.0,1.0,1.0,1.0 +hybrid+rerank,1140,10,1567.69883,1.0,1.0,1.0,1.0 +hybrid+rerank,1144,10,1982.03229,1.0,1.0,1.0,1.0 +hybrid+rerank,1146,10,1657.43742,1.0,1.0,1.0,1.0 +hybrid+rerank,1150,10,1696.84937,1.0,1.0,1.0,1.0 +hybrid+rerank,1163,10,1787.73462,1.0,1.0,1.0,1.0 +hybrid+rerank,1175,10,1826.04033,0.0,0.0,0.0,0.0 +hybrid+rerank,1179,10,1815.64225,0.0,0.0,0.0,0.0 +hybrid+rerank,1180,10,1692.61196,1.0,1.0,1.0,1.0 +hybrid+rerank,1185,10,2141.43963,1.0,1.0,1.0,1.0 +hybrid+rerank,1187,10,1793.47896,1.0,1.0,1.0,1.0 +hybrid+rerank,1191,10,1618.78787,0.0,0.0,0.0,0.0 +hybrid+rerank,1194,10,1753.14062,1.0,1.0,1.0,1.0 +hybrid+rerank,1196,10,1691.73533,0.43068,1.0,0.25,0.0 +hybrid+rerank,1197,10,1783.07375,0.43068,1.0,0.25,0.0 +hybrid+rerank,1199,10,1984.14492,0.0,0.0,0.0,0.0 +hybrid+rerank,1200,10,1935.74621,0.0,0.0,0.0,0.0 +hybrid+rerank,1202,10,1788.83104,0.63093,1.0,0.5,1.0 +hybrid+rerank,1204,10,1739.28363,0.43068,1.0,0.25,0.0 +hybrid+rerank,1207,10,1665.95154,1.0,1.0,1.0,1.0 +hybrid+rerank,1213,10,1648.99808,0.0,0.0,0.0,0.0 +hybrid+rerank,1216,10,1750.03933,1.0,1.0,1.0,1.0 +hybrid+rerank,1221,10,1853.61367,0.31546,1.0,0.125,0.0 +hybrid+rerank,1225,10,1669.35279,1.0,1.0,1.0,1.0 +hybrid+rerank,1226,10,1910.41467,0.0,0.0,0.0,0.0 +hybrid+rerank,1232,10,1940.87062,1.0,1.0,1.0,1.0 +hybrid+rerank,1241,10,1746.01446,1.0,1.0,1.0,1.0 +hybrid+rerank,1245,10,2009.14504,0.0,0.0,0.0,0.0 +hybrid+rerank,1259,10,2176.68571,1.0,1.0,1.0,1.0 +hybrid+rerank,1262,10,1588.03508,1.0,1.0,1.0,1.0 +hybrid+rerank,1266,10,1491.67237,0.5,1.0,0.33333,1.0 +hybrid+rerank,1270,10,1860.53738,1.0,1.0,1.0,1.0 +hybrid+rerank,1271,10,1821.46392,1.0,1.0,1.0,1.0 +hybrid+rerank,1272,10,1800.95825,0.5,1.0,0.33333,1.0 +hybrid+rerank,1273,10,1607.84646,1.0,1.0,1.0,1.0 +hybrid+rerank,1274,10,1686.29513,0.93252,1.0,1.0,1.0 +hybrid+rerank,1278,10,2042.30229,0.0,0.0,0.0,0.0 +hybrid+rerank,1279,10,2002.05467,0.28906,1.0,0.1,0.0 +hybrid+rerank,1280,10,1801.96192,0.0,0.0,0.0,0.0 +hybrid+rerank,1281,10,1896.63333,0.0,0.0,0.0,0.0 +hybrid+rerank,1282,10,2106.57788,1.0,1.0,1.0,1.0 +hybrid+rerank,1290,10,2012.7775,1.0,1.0,1.0,1.0 +hybrid+rerank,1292,10,1867.94567,0.5,1.0,0.33333,1.0 +hybrid+rerank,1298,10,2114.35192,1.0,1.0,1.0,1.0 +hybrid+rerank,1303,10,1939.34979,1.0,1.0,1.0,1.0 +hybrid+rerank,1316,10,1742.88963,0.0,0.0,0.0,0.0 +hybrid+rerank,1319,10,1708.16721,1.0,1.0,1.0,1.0 +hybrid+rerank,1320,10,1502.92692,1.0,1.0,1.0,1.0 +hybrid+rerank,1332,10,1768.65233,0.0,0.0,0.0,0.0 +hybrid+rerank,1335,10,1630.21571,1.0,1.0,1.0,1.0 +hybrid+rerank,1336,10,1427.86004,1.0,1.0,1.0,1.0 +hybrid+rerank,1337,10,1626.50367,0.63093,1.0,0.5,1.0 +hybrid+rerank,1339,10,1902.04538,1.0,1.0,1.0,1.0 +hybrid+rerank,1344,10,1694.55921,0.5,1.0,0.33333,1.0 +hybrid+rerank,1352,10,1988.31883,1.0,1.0,1.0,1.0 +hybrid+rerank,1359,10,2264.59563,1.0,1.0,1.0,1.0 +hybrid+rerank,1362,10,1936.28804,0.0,0.0,0.0,0.0 +hybrid+rerank,1363,10,1626.05375,0.0,0.0,0.0,0.0 +hybrid+rerank,1368,10,1548.69996,1.0,1.0,1.0,1.0 +hybrid+rerank,1370,10,1541.69537,0.63093,1.0,0.5,1.0 +hybrid+rerank,1379,10,1691.18821,0.83268,1.0,1.0,1.0 +hybrid+rerank,1382,10,1729.26863,1.0,1.0,1.0,1.0 +hybrid+rerank,1385,10,1760.07896,1.0,1.0,1.0,1.0 +hybrid+rerank,1389,10,1853.72542,1.0,1.0,1.0,1.0 +hybrid+rerank,1395,10,2103.66529,1.0,1.0,1.0,1.0 +app_default,1,0,710.89904,0.0,0.0,0.0,0.0 +app_default,3,3,413.00508,1.0,1.0,1.0,1.0 +app_default,5,1,769.69679,1.0,1.0,1.0,1.0 +app_default,13,2,579.52083,0.0,0.0,0.0,0.0 +app_default,36,3,654.99896,0.0,0.0,0.0,0.0 +app_default,42,1,605.51696,1.0,1.0,1.0,1.0 +app_default,48,2,722.91617,0.0,0.0,0.0,0.0 +app_default,49,2,588.27112,1.0,1.0,1.0,1.0 +app_default,50,3,396.56158,1.0,1.0,1.0,1.0 +app_default,51,3,722.46796,0.63093,1.0,0.5,1.0 +app_default,53,3,648.88954,1.0,1.0,1.0,1.0 +app_default,54,3,645.14383,0.0,0.0,0.0,0.0 +app_default,56,2,648.84117,1.0,1.0,1.0,1.0 +app_default,57,2,749.72825,1.0,1.0,1.0,1.0 +app_default,70,3,620.27692,0.0,0.0,0.0,0.0 +app_default,72,0,739.98546,0.0,0.0,0.0,0.0 +app_default,75,1,494.71808,1.0,1.0,1.0,1.0 +app_default,94,0,728.17463,0.0,0.0,0.0,0.0 +app_default,99,1,762.34496,1.0,1.0,1.0,1.0 +app_default,100,3,382.87029,1.0,1.0,1.0,1.0 +app_default,113,3,609.51804,1.0,1.0,1.0,1.0 +app_default,115,1,506.59,1.0,1.0,1.0,1.0 +app_default,118,3,529.55096,1.0,1.0,1.0,1.0 +app_default,124,3,632.37592,1.0,1.0,1.0,1.0 +app_default,127,2,690.65087,0.63093,1.0,0.5,1.0 +app_default,128,0,787.20508,0.0,0.0,0.0,0.0 +app_default,129,1,738.30858,1.0,1.0,1.0,1.0 +app_default,130,1,717.58429,1.0,1.0,1.0,1.0 +app_default,132,3,609.91037,0.0,0.0,0.0,0.0 +app_default,133,2,613.28221,0.55315,0.4,1.0,1.0 +app_default,137,1,769.01612,1.0,1.0,1.0,1.0 +app_default,141,3,570.54796,0.91972,1.0,1.0,1.0 +app_default,142,3,779.67083,1.0,1.0,1.0,1.0 +app_default,143,3,774.94588,1.0,1.0,1.0,1.0 +app_default,146,3,625.98388,1.0,1.0,1.0,1.0 +app_default,148,3,446.30367,1.0,1.0,1.0,1.0 +app_default,163,3,502.38696,0.63093,1.0,0.5,1.0 +app_default,171,3,553.88104,1.0,1.0,1.0,1.0 +app_default,179,3,543.27704,0.63668,0.5,1.0,1.0 +app_default,180,2,627.79458,1.0,1.0,1.0,1.0 +app_default,183,3,483.10713,1.0,1.0,1.0,1.0 +app_default,185,3,714.22571,0.5,1.0,0.33333,1.0 +app_default,198,0,807.00779,0.0,0.0,0.0,0.0 +app_default,208,3,711.49729,1.0,1.0,1.0,1.0 +app_default,212,3,692.35958,0.5,1.0,0.33333,1.0 +app_default,213,3,563.46,0.0,0.0,0.0,0.0 +app_default,216,3,715.60692,1.0,1.0,1.0,1.0 +app_default,217,3,659.84033,1.0,1.0,1.0,1.0 +app_default,218,3,587.67067,1.0,1.0,1.0,1.0 +app_default,219,3,593.44913,1.0,1.0,1.0,1.0 +app_default,230,0,665.00142,0.0,0.0,0.0,0.0 +app_default,232,3,555.54446,0.63093,1.0,0.5,1.0 +app_default,233,1,478.67146,1.0,1.0,1.0,1.0 +app_default,236,1,502.23617,1.0,1.0,1.0,1.0 +app_default,237,3,383.64687,0.0,0.0,0.0,0.0 +app_default,238,1,532.00979,0.0,0.0,0.0,0.0 +app_default,239,1,430.27671,0.0,0.0,0.0,0.0 +app_default,248,1,738.29029,1.0,1.0,1.0,1.0 +app_default,249,1,757.63317,1.0,1.0,1.0,1.0 +app_default,261,3,620.93088,0.38685,0.5,0.5,1.0 +app_default,268,2,625.38437,0.63093,1.0,0.5,1.0 +app_default,269,1,435.22237,0.0,0.0,0.0,0.0 +app_default,274,3,604.8625,1.0,1.0,1.0,1.0 +app_default,275,3,636.04067,0.70392,0.66667,1.0,1.0 +app_default,279,3,658.26746,1.0,1.0,1.0,1.0 +app_default,294,1,469.923,0.0,0.0,0.0,0.0 +app_default,295,3,359.86408,1.0,1.0,1.0,1.0 +app_default,298,3,749.65663,1.0,1.0,1.0,1.0 +app_default,300,3,716.17946,0.5,1.0,0.33333,1.0 +app_default,303,1,692.35596,0.0,0.0,0.0,0.0 +app_default,312,2,667.37988,0.0,0.0,0.0,0.0 +app_default,314,2,711.36392,0.0,0.0,0.0,0.0 +app_default,324,1,675.02358,0.0,0.0,0.0,0.0 +app_default,327,2,741.16167,1.0,1.0,1.0,1.0 +app_default,338,3,647.45313,1.0,1.0,1.0,1.0 +app_default,343,3,638.08913,0.30657,0.5,0.33333,1.0 +app_default,350,1,667.06529,1.0,1.0,1.0,1.0 +app_default,354,2,401.26521,1.0,1.0,1.0,1.0 +app_default,362,3,695.29404,1.0,1.0,1.0,1.0 +app_default,380,2,685.91967,1.0,1.0,1.0,1.0 +app_default,384,2,643.59925,0.0,0.0,0.0,0.0 +app_default,385,3,717.74629,0.38685,0.5,0.5,1.0 +app_default,386,1,705.91433,1.0,1.0,1.0,1.0 +app_default,388,0,723.90933,0.0,0.0,0.0,0.0 +app_default,399,3,588.99183,1.0,1.0,1.0,1.0 +app_default,410,3,613.20108,1.0,1.0,1.0,1.0 +app_default,411,3,485.14546,1.0,1.0,1.0,1.0 +app_default,415,3,602.91071,0.63093,1.0,0.5,1.0 +app_default,421,0,823.35983,0.0,0.0,0.0,0.0 +app_default,431,3,513.93446,0.0,0.0,0.0,0.0 +app_default,436,1,632.08083,1.0,1.0,1.0,1.0 +app_default,437,3,702.146,0.0,0.0,0.0,0.0 +app_default,439,2,713.74021,1.0,1.0,1.0,1.0 +app_default,440,1,766.0835,1.0,1.0,1.0,1.0 +app_default,443,3,591.113,1.0,1.0,1.0,1.0 +app_default,452,3,734.54308,0.38685,0.5,0.5,1.0 +app_default,475,0,642.28475,0.0,0.0,0.0,0.0 +app_default,478,1,862.87312,1.0,1.0,1.0,1.0 +app_default,491,1,739.39129,1.0,1.0,1.0,1.0 +app_default,501,2,763.54979,1.0,1.0,1.0,1.0 +app_default,502,0,752.16975,0.0,0.0,0.0,0.0 +app_default,507,3,638.62071,0.63093,1.0,0.5,1.0 +app_default,508,1,702.62608,0.0,0.0,0.0,0.0 +app_default,513,2,699.924,0.63093,1.0,0.5,1.0 +app_default,514,3,704.74846,0.5,1.0,0.33333,1.0 +app_default,516,3,586.20071,1.0,1.0,1.0,1.0 +app_default,517,3,795.43483,0.0,0.0,0.0,0.0 +app_default,521,1,549.33933,1.0,1.0,1.0,1.0 +app_default,525,3,548.22379,1.0,1.0,1.0,1.0 +app_default,527,2,717.67779,0.0,0.0,0.0,0.0 +app_default,528,3,583.8285,0.5,1.0,0.33333,1.0 +app_default,532,1,705.06571,1.0,1.0,1.0,1.0 +app_default,533,1,719.31488,1.0,1.0,1.0,1.0 +app_default,535,3,748.59825,0.0,0.0,0.0,0.0 +app_default,536,1,776.391,1.0,1.0,1.0,1.0 +app_default,539,3,511.91563,1.0,1.0,1.0,1.0 +app_default,540,3,602.20479,0.30657,0.5,0.33333,1.0 +app_default,544,2,747.77108,0.0,0.0,0.0,0.0 +app_default,549,3,744.98496,1.0,1.0,1.0,1.0 +app_default,551,1,734.66463,0.0,0.0,0.0,0.0 +app_default,552,3,394.87225,1.0,1.0,1.0,1.0 +app_default,554,3,420.15333,0.0,0.0,0.0,0.0 +app_default,560,3,694.40908,0.0,0.0,0.0,0.0 +app_default,569,3,727.59788,1.0,1.0,1.0,1.0 +app_default,575,1,788.94696,1.0,1.0,1.0,1.0 +app_default,577,0,666.72121,0.0,0.0,0.0,0.0 +app_default,578,3,779.14954,1.0,1.0,1.0,1.0 +app_default,587,1,784.01404,0.0,0.0,0.0,0.0 +app_default,589,1,730.17233,1.0,1.0,1.0,1.0 +app_default,593,2,626.78912,1.0,1.0,1.0,1.0 +app_default,597,3,473.78346,0.70392,0.66667,1.0,1.0 +app_default,598,3,424.41738,0.0,0.0,0.0,0.0 +app_default,613,1,664.9115,1.0,1.0,1.0,1.0 +app_default,619,0,803.71562,0.0,0.0,0.0,0.0 +app_default,623,3,583.37146,0.0,0.0,0.0,0.0 +app_default,628,3,727.52525,1.0,1.0,1.0,1.0 +app_default,636,3,685.03596,1.0,1.0,1.0,1.0 +app_default,637,1,774.64783,0.0,0.0,0.0,0.0 +app_default,641,1,649.32296,0.61315,0.5,1.0,1.0 +app_default,644,2,723.36954,1.0,1.0,1.0,1.0 +app_default,649,0,765.96204,0.0,0.0,0.0,0.0 +app_default,659,0,774.41746,0.0,0.0,0.0,0.0 +app_default,660,0,807.41288,0.0,0.0,0.0,0.0 +app_default,674,3,605.4275,1.0,1.0,1.0,1.0 +app_default,684,3,595.84642,0.0,0.0,0.0,0.0 +app_default,690,0,799.61146,0.0,0.0,0.0,0.0 +app_default,691,3,740.31371,0.63093,1.0,0.5,1.0 +app_default,692,1,682.43958,1.0,1.0,1.0,1.0 +app_default,693,1,772.58671,1.0,1.0,1.0,1.0 +app_default,700,1,698.99983,0.0,0.0,0.0,0.0 +app_default,702,2,704.21771,0.0,0.0,0.0,0.0 +app_default,715,3,781.39083,0.0,0.0,0.0,0.0 +app_default,716,2,827.23229,0.0,0.0,0.0,0.0 +app_default,718,0,768.04312,0.0,0.0,0.0,0.0 +app_default,721,1,780.13275,1.0,1.0,1.0,1.0 +app_default,723,1,410.21583,1.0,1.0,1.0,1.0 +app_default,727,2,743.2585,0.0,0.0,0.0,0.0 +app_default,728,3,708.44729,0.61315,0.5,1.0,1.0 +app_default,729,2,753.06217,1.0,1.0,1.0,1.0 +app_default,742,2,787.83425,1.0,1.0,1.0,1.0 +app_default,743,2,814.36083,1.0,1.0,1.0,1.0 +app_default,744,3,363.45696,0.63093,1.0,0.5,1.0 +app_default,756,3,723.89579,0.5,1.0,0.33333,1.0 +app_default,759,3,579.48396,1.0,1.0,1.0,1.0 +app_default,768,1,814.49242,0.0,0.0,0.0,0.0 +app_default,770,2,699.36254,1.0,1.0,1.0,1.0 +app_default,775,3,407.95508,0.0,0.0,0.0,0.0 +app_default,781,3,537.958,0.63093,1.0,0.5,1.0 +app_default,783,3,796.86204,0.0,0.0,0.0,0.0 +app_default,784,3,874.59608,1.0,1.0,1.0,1.0 +app_default,785,1,913.38913,0.0,0.0,0.0,0.0 +app_default,793,2,725.96292,1.0,1.0,1.0,1.0 +app_default,800,3,729.81962,0.63093,1.0,0.5,1.0 +app_default,805,3,571.85358,1.0,1.0,1.0,1.0 +app_default,808,1,745.03021,1.0,1.0,1.0,1.0 +app_default,811,3,784.61292,1.0,1.0,1.0,1.0 +app_default,814,2,760.47513,1.0,1.0,1.0,1.0 +app_default,820,0,706.38912,0.0,0.0,0.0,0.0 +app_default,821,0,880.93133,0.0,0.0,0.0,0.0 +app_default,823,3,742.058,1.0,1.0,1.0,1.0 +app_default,830,1,776.56558,0.0,0.0,0.0,0.0 +app_default,831,0,815.42025,0.0,0.0,0.0,0.0 +app_default,832,3,824.56333,1.0,1.0,1.0,1.0 +app_default,834,1,852.11958,0.0,0.0,0.0,0.0 +app_default,837,3,670.46383,0.63093,1.0,0.5,1.0 +app_default,839,3,717.87083,1.0,1.0,1.0,1.0 +app_default,845,3,467.56258,1.0,1.0,1.0,1.0 +app_default,847,2,620.88642,1.0,1.0,1.0,1.0 +app_default,852,3,665.52158,0.63093,1.0,0.5,1.0 +app_default,859,3,505.71846,0.63093,1.0,0.5,1.0 +app_default,870,2,612.82421,0.0,0.0,0.0,0.0 +app_default,873,3,861.677,0.0,0.0,0.0,0.0 +app_default,879,1,909.55642,1.0,1.0,1.0,1.0 +app_default,880,1,973.89579,1.0,1.0,1.0,1.0 +app_default,882,1,572.01079,1.0,1.0,1.0,1.0 +app_default,887,0,903.66167,0.0,0.0,0.0,0.0 +app_default,903,3,816.04487,1.0,1.0,1.0,1.0 +app_default,904,2,919.36633,1.0,1.0,1.0,1.0 +app_default,907,3,713.98417,1.0,1.0,1.0,1.0 +app_default,911,3,819.69567,1.0,1.0,1.0,1.0 +app_default,913,3,873.49112,0.0,0.0,0.0,0.0 +app_default,914,3,903.33104,0.0,0.0,0.0,0.0 +app_default,921,3,894.24017,1.0,1.0,1.0,1.0 +app_default,922,1,906.97462,1.0,1.0,1.0,1.0 +app_default,936,1,850.9535,1.0,1.0,1.0,1.0 +app_default,956,3,743.76733,1.0,1.0,1.0,1.0 +app_default,957,1,457.60063,1.0,1.0,1.0,1.0 +app_default,960,1,916.61479,1.0,1.0,1.0,1.0 +app_default,967,3,895.10279,0.91972,1.0,1.0,1.0 +app_default,971,3,466.50525,0.63668,0.5,1.0,1.0 +app_default,975,3,852.05087,0.0,0.0,0.0,0.0 +app_default,982,1,1046.89938,1.0,1.0,1.0,1.0 +app_default,985,3,817.98775,1.0,1.0,1.0,1.0 +app_default,993,3,695.41554,1.0,1.0,1.0,1.0 +app_default,1012,3,708.15521,1.0,1.0,1.0,1.0 +app_default,1014,1,883.168,1.0,1.0,1.0,1.0 +app_default,1019,2,783.92033,0.63093,1.0,0.5,1.0 +app_default,1020,2,760.8055,0.63093,1.0,0.5,1.0 +app_default,1021,1,764.42837,0.0,0.0,0.0,0.0 +app_default,1024,2,780.16996,1.0,1.0,1.0,1.0 +app_default,1029,3,885.79921,0.76536,0.66667,1.0,1.0 +app_default,1041,3,664.58283,0.0,0.0,0.0,0.0 +app_default,1049,0,764.43196,0.0,0.0,0.0,0.0 +app_default,1062,3,926.00329,1.0,1.0,1.0,1.0 +app_default,1086,1,930.66396,1.0,1.0,1.0,1.0 +app_default,1088,3,859.65596,1.0,1.0,1.0,1.0 +app_default,1089,2,670.68075,1.0,1.0,1.0,1.0 +app_default,1099,3,539.58233,0.0,0.0,0.0,0.0 +app_default,1100,3,495.8155,0.5,1.0,0.33333,1.0 +app_default,1104,2,565.04571,1.0,1.0,1.0,1.0 +app_default,1107,3,525.30162,1.0,1.0,1.0,1.0 +app_default,1110,0,743.51863,0.0,0.0,0.0,0.0 +app_default,1121,3,571.05683,1.0,1.0,1.0,1.0 +app_default,1130,3,574.50225,1.0,1.0,1.0,1.0 +app_default,1132,3,579.45783,0.61315,0.5,1.0,1.0 +app_default,1137,3,617.57725,1.0,1.0,1.0,1.0 +app_default,1140,0,542.29375,0.0,0.0,0.0,0.0 +app_default,1144,2,578.59062,1.0,1.0,1.0,1.0 +app_default,1146,1,708.45825,1.0,1.0,1.0,1.0 +app_default,1150,3,667.89108,1.0,1.0,1.0,1.0 +app_default,1163,2,526.11475,1.0,1.0,1.0,1.0 +app_default,1175,0,643.66792,0.0,0.0,0.0,0.0 +app_default,1179,1,637.13617,0.0,0.0,0.0,0.0 +app_default,1180,3,656.05108,0.0,0.0,0.0,0.0 +app_default,1185,1,588.37892,1.0,1.0,1.0,1.0 +app_default,1187,3,685.66883,1.0,1.0,1.0,1.0 +app_default,1191,0,821.88167,0.0,0.0,0.0,0.0 +app_default,1194,1,769.64971,1.0,1.0,1.0,1.0 +app_default,1196,0,730.60621,0.0,0.0,0.0,0.0 +app_default,1197,0,736.59813,0.0,0.0,0.0,0.0 +app_default,1199,2,646.26733,0.0,0.0,0.0,0.0 +app_default,1200,0,757.19967,0.0,0.0,0.0,0.0 +app_default,1202,3,579.84438,0.63093,1.0,0.5,1.0 +app_default,1204,1,724.14492,0.0,0.0,0.0,0.0 +app_default,1207,1,808.78025,1.0,1.0,1.0,1.0 +app_default,1213,3,540.08163,0.0,0.0,0.0,0.0 +app_default,1216,1,693.947,1.0,1.0,1.0,1.0 +app_default,1221,0,755.44562,0.0,0.0,0.0,0.0 +app_default,1225,2,718.55158,1.0,1.0,1.0,1.0 +app_default,1226,1,695.23721,0.0,0.0,0.0,0.0 +app_default,1232,1,567.91683,1.0,1.0,1.0,1.0 +app_default,1241,3,697.47492,1.0,1.0,1.0,1.0 +app_default,1245,0,708.46496,0.0,0.0,0.0,0.0 +app_default,1259,3,573.31092,1.0,1.0,1.0,1.0 +app_default,1262,3,341.16463,1.0,1.0,1.0,1.0 +app_default,1266,3,545.09392,0.5,1.0,0.33333,1.0 +app_default,1270,1,556.89196,1.0,1.0,1.0,1.0 +app_default,1271,1,547.14833,1.0,1.0,1.0,1.0 +app_default,1272,0,660.04,0.0,0.0,0.0,0.0 +app_default,1273,3,581.79471,1.0,1.0,1.0,1.0 +app_default,1274,3,466.90596,0.76536,0.66667,1.0,1.0 +app_default,1278,0,711.62471,0.0,0.0,0.0,0.0 +app_default,1279,1,669.28946,0.0,0.0,0.0,0.0 +app_default,1280,3,587.39379,0.0,0.0,0.0,0.0 +app_default,1281,1,600.80612,0.0,0.0,0.0,0.0 +app_default,1282,1,655.88812,1.0,1.0,1.0,1.0 +app_default,1290,1,637.06587,1.0,1.0,1.0,1.0 +app_default,1292,3,611.86692,0.63093,1.0,0.5,1.0 +app_default,1298,1,561.29708,1.0,1.0,1.0,1.0 +app_default,1303,0,653.83454,0.0,0.0,0.0,0.0 +app_default,1316,1,636.11296,0.0,0.0,0.0,0.0 +app_default,1319,3,593.17446,1.0,1.0,1.0,1.0 +app_default,1320,3,567.89046,1.0,1.0,1.0,1.0 +app_default,1332,3,631.60187,0.0,0.0,0.0,0.0 +app_default,1335,3,511.08204,1.0,1.0,1.0,1.0 +app_default,1336,3,539.93029,1.0,1.0,1.0,1.0 +app_default,1337,3,713.10454,1.0,1.0,1.0,1.0 +app_default,1339,1,611.36183,1.0,1.0,1.0,1.0 +app_default,1344,3,601.67967,0.63093,1.0,0.5,1.0 +app_default,1352,2,553.84667,1.0,1.0,1.0,1.0 +app_default,1359,3,535.67854,1.0,1.0,1.0,1.0 +app_default,1362,0,653.35379,0.0,0.0,0.0,0.0 +app_default,1363,0,546.2155,0.0,0.0,0.0,0.0 +app_default,1368,3,538.06521,0.0,0.0,0.0,0.0 +app_default,1370,2,553.25604,0.63093,1.0,0.5,1.0 +app_default,1379,3,596.92317,0.39038,0.25,1.0,1.0 +app_default,1382,3,590.72392,1.0,1.0,1.0,1.0 +app_default,1385,2,588.92204,1.0,1.0,1.0,1.0 +app_default,1389,3,630.72663,1.0,1.0,1.0,1.0 +app_default,1395,0,613.716,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1,3,661.42662,0.0,0.0,0.0,0.0 +app_default_no_cutoff,3,3,412.73037,1.0,1.0,1.0,1.0 +app_default_no_cutoff,5,3,692.40758,1.0,1.0,1.0,1.0 +app_default_no_cutoff,13,3,591.92913,0.0,0.0,0.0,0.0 +app_default_no_cutoff,36,3,614.27683,0.0,0.0,0.0,0.0 +app_default_no_cutoff,42,3,605.76646,1.0,1.0,1.0,1.0 +app_default_no_cutoff,48,3,688.8775,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,49,3,554.64387,1.0,1.0,1.0,1.0 +app_default_no_cutoff,50,3,410.83317,1.0,1.0,1.0,1.0 +app_default_no_cutoff,51,3,686.34754,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,53,3,666.61983,1.0,1.0,1.0,1.0 +app_default_no_cutoff,54,3,646.58879,0.0,0.0,0.0,0.0 +app_default_no_cutoff,56,3,659.64554,1.0,1.0,1.0,1.0 +app_default_no_cutoff,57,3,614.01013,1.0,1.0,1.0,1.0 +app_default_no_cutoff,70,3,714.82338,0.0,0.0,0.0,0.0 +app_default_no_cutoff,72,3,691.35904,1.0,1.0,1.0,1.0 +app_default_no_cutoff,75,3,522.97204,1.0,1.0,1.0,1.0 +app_default_no_cutoff,94,3,753.67462,0.0,0.0,0.0,0.0 +app_default_no_cutoff,99,3,725.78983,1.0,1.0,1.0,1.0 +app_default_no_cutoff,100,3,382.07196,1.0,1.0,1.0,1.0 +app_default_no_cutoff,113,3,492.82971,1.0,1.0,1.0,1.0 +app_default_no_cutoff,115,3,511.35883,1.0,1.0,1.0,1.0 +app_default_no_cutoff,118,3,508.09275,1.0,1.0,1.0,1.0 +app_default_no_cutoff,124,3,576.33842,1.0,1.0,1.0,1.0 +app_default_no_cutoff,127,3,605.493,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,128,3,741.06175,0.0,0.0,0.0,0.0 +app_default_no_cutoff,129,3,699.73596,1.0,1.0,1.0,1.0 +app_default_no_cutoff,130,3,612.16754,1.0,1.0,1.0,1.0 +app_default_no_cutoff,132,3,531.18413,0.0,0.0,0.0,0.0 +app_default_no_cutoff,133,3,609.04517,0.55315,0.4,1.0,1.0 +app_default_no_cutoff,137,3,674.72425,1.0,1.0,1.0,1.0 +app_default_no_cutoff,141,3,536.12446,0.91972,1.0,1.0,1.0 +app_default_no_cutoff,142,3,704.71667,1.0,1.0,1.0,1.0 +app_default_no_cutoff,143,3,610.44433,1.0,1.0,1.0,1.0 +app_default_no_cutoff,146,3,598.74771,1.0,1.0,1.0,1.0 +app_default_no_cutoff,148,3,500.76792,1.0,1.0,1.0,1.0 +app_default_no_cutoff,163,3,477.863,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,171,3,501.54221,1.0,1.0,1.0,1.0 +app_default_no_cutoff,179,3,616.68154,0.63668,0.5,1.0,1.0 +app_default_no_cutoff,180,3,605.97975,1.0,1.0,1.0,1.0 +app_default_no_cutoff,183,3,489.16271,1.0,1.0,1.0,1.0 +app_default_no_cutoff,185,3,711.17988,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,198,3,750.585,0.0,0.0,0.0,0.0 +app_default_no_cutoff,208,3,671.97237,1.0,1.0,1.0,1.0 +app_default_no_cutoff,212,3,641.60229,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,213,3,527.56321,0.0,0.0,0.0,0.0 +app_default_no_cutoff,216,3,651.08287,1.0,1.0,1.0,1.0 +app_default_no_cutoff,217,3,669.32375,1.0,1.0,1.0,1.0 +app_default_no_cutoff,218,3,574.71825,1.0,1.0,1.0,1.0 +app_default_no_cutoff,219,3,569.46962,1.0,1.0,1.0,1.0 +app_default_no_cutoff,230,3,609.60588,1.0,1.0,1.0,1.0 +app_default_no_cutoff,232,3,585.51392,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,233,3,464.41742,1.0,1.0,1.0,1.0 +app_default_no_cutoff,236,3,505.57183,1.0,1.0,1.0,1.0 +app_default_no_cutoff,237,3,382.62842,0.0,0.0,0.0,0.0 +app_default_no_cutoff,238,3,537.83208,0.0,0.0,0.0,0.0 +app_default_no_cutoff,239,3,432.74279,0.0,0.0,0.0,0.0 +app_default_no_cutoff,248,3,656.80879,1.0,1.0,1.0,1.0 +app_default_no_cutoff,249,3,695.05417,1.0,1.0,1.0,1.0 +app_default_no_cutoff,261,3,550.45325,0.38685,0.5,0.5,1.0 +app_default_no_cutoff,268,3,581.96612,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,269,3,437.79704,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,274,3,538.19538,1.0,1.0,1.0,1.0 +app_default_no_cutoff,275,3,588.76967,0.70392,0.66667,1.0,1.0 +app_default_no_cutoff,279,3,628.24067,1.0,1.0,1.0,1.0 +app_default_no_cutoff,294,3,478.12871,0.0,0.0,0.0,0.0 +app_default_no_cutoff,295,3,366.877,1.0,1.0,1.0,1.0 +app_default_no_cutoff,298,3,674.83246,1.0,1.0,1.0,1.0 +app_default_no_cutoff,300,3,686.10167,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,303,3,670.12071,0.0,0.0,0.0,0.0 +app_default_no_cutoff,312,3,641.30946,0.0,0.0,0.0,0.0 +app_default_no_cutoff,314,3,654.12512,0.0,0.0,0.0,0.0 +app_default_no_cutoff,324,3,640.54821,0.0,0.0,0.0,0.0 +app_default_no_cutoff,327,3,682.76237,1.0,1.0,1.0,1.0 +app_default_no_cutoff,338,3,580.7045,1.0,1.0,1.0,1.0 +app_default_no_cutoff,343,3,630.40079,0.30657,0.5,0.33333,1.0 +app_default_no_cutoff,350,3,543.01104,1.0,1.0,1.0,1.0 +app_default_no_cutoff,354,3,417.34137,1.0,1.0,1.0,1.0 +app_default_no_cutoff,362,3,633.80542,1.0,1.0,1.0,1.0 +app_default_no_cutoff,380,3,649.26708,1.0,1.0,1.0,1.0 +app_default_no_cutoff,384,3,649.81896,0.0,0.0,0.0,0.0 +app_default_no_cutoff,385,3,641.60812,0.38685,0.5,0.5,1.0 +app_default_no_cutoff,386,3,600.67646,1.0,1.0,1.0,1.0 +app_default_no_cutoff,388,3,656.5535,0.0,0.0,0.0,0.0 +app_default_no_cutoff,399,3,541.29325,1.0,1.0,1.0,1.0 +app_default_no_cutoff,410,3,593.147,1.0,1.0,1.0,1.0 +app_default_no_cutoff,411,3,558.16821,1.0,1.0,1.0,1.0 +app_default_no_cutoff,415,3,559.87658,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,421,3,763.41313,0.0,0.0,0.0,0.0 +app_default_no_cutoff,431,3,525.76725,0.0,0.0,0.0,0.0 +app_default_no_cutoff,436,3,590.65533,1.0,1.0,1.0,1.0 +app_default_no_cutoff,437,3,613.55746,0.0,0.0,0.0,0.0 +app_default_no_cutoff,439,3,673.24492,1.0,1.0,1.0,1.0 +app_default_no_cutoff,440,3,713.1925,1.0,1.0,1.0,1.0 +app_default_no_cutoff,443,3,560.24858,1.0,1.0,1.0,1.0 +app_default_no_cutoff,452,3,675.06437,0.38685,0.5,0.5,1.0 +app_default_no_cutoff,475,3,569.73133,0.0,0.0,0.0,0.0 +app_default_no_cutoff,478,3,691.6615,1.0,1.0,1.0,1.0 +app_default_no_cutoff,491,3,688.55929,1.0,1.0,1.0,1.0 +app_default_no_cutoff,501,3,610.68758,1.0,1.0,1.0,1.0 +app_default_no_cutoff,502,3,759.93779,0.0,0.0,0.0,0.0 +app_default_no_cutoff,507,3,646.48187,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,508,3,570.922,0.0,0.0,0.0,0.0 +app_default_no_cutoff,513,3,651.39821,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,514,3,675.34167,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,516,3,525.24021,1.0,1.0,1.0,1.0 +app_default_no_cutoff,517,3,673.07658,0.0,0.0,0.0,0.0 +app_default_no_cutoff,521,3,536.17337,1.0,1.0,1.0,1.0 +app_default_no_cutoff,525,3,495.46463,1.0,1.0,1.0,1.0 +app_default_no_cutoff,527,3,663.82708,0.0,0.0,0.0,0.0 +app_default_no_cutoff,528,3,690.07542,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,532,3,640.88021,1.0,1.0,1.0,1.0 +app_default_no_cutoff,533,3,657.0045,1.0,1.0,1.0,1.0 +app_default_no_cutoff,535,3,690.3485,0.0,0.0,0.0,0.0 +app_default_no_cutoff,536,3,700.90629,1.0,1.0,1.0,1.0 +app_default_no_cutoff,539,3,571.41883,1.0,1.0,1.0,1.0 +app_default_no_cutoff,540,3,568.50417,0.30657,0.5,0.33333,1.0 +app_default_no_cutoff,544,3,643.2575,0.0,0.0,0.0,0.0 +app_default_no_cutoff,549,3,679.53804,1.0,1.0,1.0,1.0 +app_default_no_cutoff,551,3,670.201,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,552,3,419.34075,1.0,1.0,1.0,1.0 +app_default_no_cutoff,554,3,421.16383,0.0,0.0,0.0,0.0 +app_default_no_cutoff,560,3,652.08975,0.0,0.0,0.0,0.0 +app_default_no_cutoff,569,3,650.15292,1.0,1.0,1.0,1.0 +app_default_no_cutoff,575,3,700.678,1.0,1.0,1.0,1.0 +app_default_no_cutoff,577,3,673.2165,0.0,0.0,0.0,0.0 +app_default_no_cutoff,578,3,708.07217,1.0,1.0,1.0,1.0 +app_default_no_cutoff,587,3,715.93071,0.0,0.0,0.0,0.0 +app_default_no_cutoff,589,3,596.84192,1.0,1.0,1.0,1.0 +app_default_no_cutoff,593,3,586.42108,1.0,1.0,1.0,1.0 +app_default_no_cutoff,597,3,421.77946,0.70392,0.66667,1.0,1.0 +app_default_no_cutoff,598,3,493.85175,0.0,0.0,0.0,0.0 +app_default_no_cutoff,613,3,670.4765,1.0,1.0,1.0,1.0 +app_default_no_cutoff,619,3,693.85325,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,623,3,577.43721,0.0,0.0,0.0,0.0 +app_default_no_cutoff,628,3,640.36154,1.0,1.0,1.0,1.0 +app_default_no_cutoff,636,3,640.24075,1.0,1.0,1.0,1.0 +app_default_no_cutoff,637,3,695.04525,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,641,3,642.77437,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,644,3,689.08087,1.0,1.0,1.0,1.0 +app_default_no_cutoff,649,3,689.08433,1.0,1.0,1.0,1.0 +app_default_no_cutoff,659,3,731.576,0.0,0.0,0.0,0.0 +app_default_no_cutoff,660,3,765.07546,0.0,0.0,0.0,0.0 +app_default_no_cutoff,674,3,580.72354,1.0,1.0,1.0,1.0 +app_default_no_cutoff,684,3,584.21271,0.0,0.0,0.0,0.0 +app_default_no_cutoff,690,3,746.4585,0.0,0.0,0.0,0.0 +app_default_no_cutoff,691,3,619.90546,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,692,3,598.23892,1.0,1.0,1.0,1.0 +app_default_no_cutoff,693,3,683.77796,1.0,1.0,1.0,1.0 +app_default_no_cutoff,700,3,587.43129,0.0,0.0,0.0,0.0 +app_default_no_cutoff,702,3,577.62275,0.0,0.0,0.0,0.0 +app_default_no_cutoff,715,3,707.73587,0.0,0.0,0.0,0.0 +app_default_no_cutoff,716,3,727.84879,0.0,0.0,0.0,0.0 +app_default_no_cutoff,718,3,601.61988,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,721,3,729.714,1.0,1.0,1.0,1.0 +app_default_no_cutoff,723,3,425.99242,1.0,1.0,1.0,1.0 +app_default_no_cutoff,727,3,677.70021,0.0,0.0,0.0,0.0 +app_default_no_cutoff,728,3,672.32308,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,729,3,661.08858,1.0,1.0,1.0,1.0 +app_default_no_cutoff,742,3,678.17129,1.0,1.0,1.0,1.0 +app_default_no_cutoff,743,3,757.14883,1.0,1.0,1.0,1.0 +app_default_no_cutoff,744,3,388.81188,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,756,3,685.86521,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,759,3,568.30342,1.0,1.0,1.0,1.0 +app_default_no_cutoff,768,3,644.45446,0.0,0.0,0.0,0.0 +app_default_no_cutoff,770,3,738.96208,1.0,1.0,1.0,1.0 +app_default_no_cutoff,775,3,426.87179,0.0,0.0,0.0,0.0 +app_default_no_cutoff,781,3,535.95058,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,783,3,629.36896,0.0,0.0,0.0,0.0 +app_default_no_cutoff,784,3,726.54204,1.0,1.0,1.0,1.0 +app_default_no_cutoff,785,3,685.99963,0.0,0.0,0.0,0.0 +app_default_no_cutoff,793,3,695.215,1.0,1.0,1.0,1.0 +app_default_no_cutoff,800,3,654.8685,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,805,3,530.95729,1.0,1.0,1.0,1.0 +app_default_no_cutoff,808,3,747.50567,1.0,1.0,1.0,1.0 +app_default_no_cutoff,811,3,672.61304,1.0,1.0,1.0,1.0 +app_default_no_cutoff,814,3,705.85946,1.0,1.0,1.0,1.0 +app_default_no_cutoff,820,3,677.4335,0.0,0.0,0.0,0.0 +app_default_no_cutoff,821,3,781.443,0.0,0.0,0.0,0.0 +app_default_no_cutoff,823,3,605.64175,1.0,1.0,1.0,1.0 +app_default_no_cutoff,830,3,605.78179,0.0,0.0,0.0,0.0 +app_default_no_cutoff,831,3,603.52912,0.0,0.0,0.0,0.0 +app_default_no_cutoff,832,3,692.61454,1.0,1.0,1.0,1.0 +app_default_no_cutoff,834,3,732.00237,0.0,0.0,0.0,0.0 +app_default_no_cutoff,837,3,687.50408,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,839,3,670.50875,1.0,1.0,1.0,1.0 +app_default_no_cutoff,845,3,420.87879,1.0,1.0,1.0,1.0 +app_default_no_cutoff,847,3,521.32863,1.0,1.0,1.0,1.0 +app_default_no_cutoff,852,3,656.36475,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,859,3,478.05458,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,870,3,629.67396,0.0,0.0,0.0,0.0 +app_default_no_cutoff,873,3,691.17058,0.0,0.0,0.0,0.0 +app_default_no_cutoff,879,3,749.61833,1.0,1.0,1.0,1.0 +app_default_no_cutoff,880,3,633.70242,1.0,1.0,1.0,1.0 +app_default_no_cutoff,882,3,651.36875,1.0,1.0,1.0,1.0 +app_default_no_cutoff,887,3,703.08296,0.0,0.0,0.0,0.0 +app_default_no_cutoff,903,3,674.875,1.0,1.0,1.0,1.0 +app_default_no_cutoff,904,3,738.79071,1.0,1.0,1.0,1.0 +app_default_no_cutoff,907,3,608.49983,1.0,1.0,1.0,1.0 +app_default_no_cutoff,911,3,675.12696,1.0,1.0,1.0,1.0 +app_default_no_cutoff,913,3,827.28,0.0,0.0,0.0,0.0 +app_default_no_cutoff,914,3,825.54763,0.0,0.0,0.0,0.0 +app_default_no_cutoff,921,3,707.12796,1.0,1.0,1.0,1.0 +app_default_no_cutoff,922,3,696.73108,1.0,1.0,1.0,1.0 +app_default_no_cutoff,936,3,787.27483,1.0,1.0,1.0,1.0 +app_default_no_cutoff,956,3,689.51304,1.0,1.0,1.0,1.0 +app_default_no_cutoff,957,3,464.17588,1.0,1.0,1.0,1.0 +app_default_no_cutoff,960,3,697.03742,1.0,1.0,1.0,1.0 +app_default_no_cutoff,967,3,685.03842,0.91972,1.0,1.0,1.0 +app_default_no_cutoff,971,3,453.58792,0.63668,0.5,1.0,1.0 +app_default_no_cutoff,975,3,703.25463,0.0,0.0,0.0,0.0 +app_default_no_cutoff,982,3,752.03838,1.0,1.0,1.0,1.0 +app_default_no_cutoff,985,3,580.68954,1.0,1.0,1.0,1.0 +app_default_no_cutoff,993,3,655.42113,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1012,3,437.208,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1014,3,689.09217,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1019,3,708.11596,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1020,3,636.65413,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1021,3,641.75283,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1024,3,767.49383,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1029,3,637.22663,0.76536,0.66667,1.0,1.0 +app_default_no_cutoff,1041,3,566.979,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1049,3,675.98608,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1062,3,693.94342,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1086,3,706.56612,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1088,3,788.46125,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1089,3,572.20083,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1099,3,599.21254,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1100,3,604.13821,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,1104,3,726.58854,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1107,3,541.86913,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1110,3,817.30971,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1121,3,617.13517,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1130,3,729.17558,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1132,3,703.94196,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,1137,3,767.5925,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1140,3,625.3885,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1144,3,691.47558,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1146,3,722.41179,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1150,3,762.40363,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1163,3,533.13429,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1175,3,743.61708,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1179,3,764.321,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1180,3,760.63167,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1185,3,642.74337,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1187,3,759.79654,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1191,3,798.82108,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1194,3,749.88425,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1196,3,792.60512,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1197,3,825.73429,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1199,3,681.10696,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1200,3,864.36621,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1202,3,670.10971,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1204,3,721.57904,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1207,3,747.98883,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1213,3,620.71879,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1216,3,778.00133,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1221,3,843.88625,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1225,3,767.41279,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1226,3,810.311,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1232,3,693.73388,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1241,3,832.90171,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1245,3,819.10758,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1259,3,594.73804,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1262,3,356.60171,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1266,3,531.76583,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,1270,3,652.63263,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1271,3,631.33079,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1272,3,802.23633,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1273,3,690.74162,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1274,3,461.39071,0.76536,0.66667,1.0,1.0 +app_default_no_cutoff,1278,3,804.65137,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1279,3,653.21575,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1280,3,698.22313,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1281,3,713.03458,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1282,3,815.88125,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1290,3,750.12575,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1292,3,738.55246,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1298,3,617.0095,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1303,3,807.25171,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1316,3,752.78592,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1319,3,730.94258,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1320,3,666.38458,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1332,3,712.42762,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1335,3,584.11512,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1336,3,626.24792,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1337,3,803.47167,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1339,3,801.06663,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1344,3,715.76254,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1352,3,712.08237,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1359,3,629.71896,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1362,3,796.93971,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1363,3,580.59117,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1368,3,602.16313,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1370,3,546.15554,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1379,3,707.69267,0.39038,0.25,1.0,1.0 +app_default_no_cutoff,1382,3,722.6655,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1385,3,607.096,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1389,3,671.60225,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1395,3,722.11004,1.0,1.0,1.0,1.0 diff --git a/evaluation/results.json b/evaluation/results.json new file mode 100644 index 0000000..635f8f3 --- /dev/null +++ b/evaluation/results.json @@ -0,0 +1,84 @@ +{ + "scifact": { + "dataset": "scifact", + "documents_in_corpus": 5183, + "documents_stored": 5183, + "test_questions": 300, + "full_test_set": true, + "sources": { + "corpus": [ + "BeIR/scifact", + "b3b5335604bf5ee3c4447671af975ea25143d4f5" + ], + "qrels": [ + "BeIR/scifact-qrels", + "2938d17dc3b09882fdb8c12bbbe2e2dc0e75a029" + ], + "embedder": [ + "sentence-transformers/all-MiniLM-L6-v2", + "1110a243fdf4706b3f48f1d95db1a4f5529b4d41" + ], + "reranker": [ + "cross-encoder/ms-marco-MiniLM-L-6-v2", + "233902d25c440f23af6f7d6e94d2946bac0bee0a" + ] + }, + "machine": "arm64, Darwin 25.4.0, CPU only", + "packages": { + "chromadb": "1.5.9", + "sentence-transformers": "6.3.0", + "torch": "2.15.0", + "rank-bm25": "0.2.2" + }, + "setups": { + "dense": { + "ndcg@10": 0.6451, + "recall@10": 0.7833, + "mrr@10": 0.6047, + "hit@3": 0.6833, + "no_passage_share": 0.0, + "ms_per_question_median": 9.5 + }, + "bm25": { + "ndcg@10": 0.5597, + "recall@10": 0.6862, + "mrr@10": 0.5242, + "hit@3": 0.59, + "no_passage_share": 0.0, + "ms_per_question_median": 10.5 + }, + "dense+rerank": { + "ndcg@10": 0.6909, + "recall@10": 0.8102, + "mrr@10": 0.6619, + "hit@3": 0.71, + "no_passage_share": 0.0, + "ms_per_question_median": 1225.1 + }, + "hybrid+rerank": { + "ndcg@10": 0.6935, + "recall@10": 0.8172, + "mrr@10": 0.6622, + "hit@3": 0.72, + "no_passage_share": 0.0, + "ms_per_question_median": 1871.6 + }, + "app_default": { + "ndcg@10": 0.6029, + "recall@10": 0.6411, + "mrr@10": 0.6044, + "hit@3": 0.6667, + "no_passage_share": 0.1233, + "ms_per_question_median": 678.7 + }, + "app_default_no_cutoff": { + "ndcg@10": 0.6392, + "recall@10": 0.6861, + "mrr@10": 0.6389, + "hit@3": 0.7133, + "no_passage_share": 0.0, + "ms_per_question_median": 655.9 + } + } + } +} diff --git a/evaluation/retrieval_eval.py b/evaluation/retrieval_eval.py new file mode 100644 index 0000000..4953c6e --- /dev/null +++ b/evaluation/retrieval_eval.py @@ -0,0 +1,209 @@ +"""Measure how well the app's search finds the right passage, on public BEIR benchmarks. + +Every document of the benchmark is stored as one passage in the app's own engine +(`src/rag_engine.py`, ChromaDB backend). The steps that read files and split them into +chunks are not part of this test. Then every test question goes through these set-ups: + + dense vector search only (all-MiniLM-L6-v2 in ChromaDB) + bm25 keyword search only (the app's BM25) + dense+rerank the app's retrieve() with BM25 switched off: vector candidates, then the + cross-encoder puts them in order + hybrid+rerank the app's retrieve() as it is: vector and keyword candidates, then the + cross-encoder (top_k=10, so 30 + 30 candidates; no confidence cut-off) + +Scores: nDCG@10 (the standard BEIR score), Recall@10, MRR@10, and Hit@3 (a relevant passage +is among the first 3). + +Then the app's default call, retrieve(question), which keeps 3 passages and drops passages +whose reranker confidence is below 0.3. When nothing is left the app does not answer. This is +scored with Hit@3 (no passage counts as a miss) and the share of questions with no passage, +and compared with the same call without the cut-off. + +Run from the repository root: + python -m evaluation.retrieval_eval --dataset scifact +""" + +import argparse +import json +import platform +import statistics +import sys +import tempfile +import time +from importlib.metadata import version +from pathlib import Path + +import pandas as pd +from huggingface_hub import hf_hub_download +from sentence_transformers import CrossEncoder, SentenceTransformer + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from rag_engine import RAGEngine # noqa: E402 + +from evaluation.metrics import hit_at_k, mrr_at_k, ndcg_at_k, recall_at_k # noqa: E402 + +# Pinned versions, so the numbers can be reproduced. +DATASETS = { + "scifact": { + "corpus": ("BeIR/scifact", "b3b5335604bf5ee3c4447671af975ea25143d4f5"), + "qrels": ("BeIR/scifact-qrels", "2938d17dc3b09882fdb8c12bbbe2e2dc0e75a029"), + }, + "fiqa": { + "corpus": ("BeIR/fiqa", "979c07a7cb5ccc6ca009792241fa1250b98055dd"), + "qrels": ("BeIR/fiqa-qrels", "252958f2d646e22cab6d0c72dd3f0d5de6d0655a"), + }, +} +EMBEDDER = ("sentence-transformers/all-MiniLM-L6-v2", "1110a243fdf4706b3f48f1d95db1a4f5529b4d41") +RERANKER = ("cross-encoder/ms-marco-MiniLM-L-6-v2", "233902d25c440f23af6f7d6e94d2946bac0bee0a") +K = 10 +APP_TOP_K = 3 +OUT_DIR = Path(__file__).resolve().parent + + +def load(name: str, split: str = "test") -> tuple[pd.DataFrame, dict[str, str], dict[str, dict[str, int]]]: + """Corpus, test questions and their relevant documents for one BEIR dataset.""" + (crepo, crev), (qrepo, qrev) = DATASETS[name]["corpus"], DATASETS[name]["qrels"] + + def get(repo, file, rev): + return hf_hub_download(repo, file, repo_type="dataset", revision=rev) + + corpus = pd.read_parquet(get(crepo, "corpus/corpus-00000-of-00001.parquet", crev)) + queries = pd.read_parquet(get(crepo, "queries/queries-00000-of-00001.parquet", crev)) + qrels = pd.read_csv(get(qrepo, f"{split}.tsv", qrev), sep="\t", dtype=str) + qrels["score"] = qrels["score"].astype(int) + + relevant: dict[str, dict[str, int]] = {} + for row in qrels.itertuples(index=False): + relevant.setdefault(row[0], {})[row[1]] = row[2] + qtext = dict(zip(queries["_id"].astype(str), queries["text"])) + questions = {qid: qtext[qid] for qid in relevant if qid in qtext} + return corpus, questions, relevant + + +def build_engine(corpus: pd.DataFrame, embedder, reranker, db_path: str, name: str) -> tuple[RAGEngine, int]: + """Store every document as one passage, with its id as the source name.""" + engine = RAGEngine(db_path=db_path, collection_name=f"beir_{name}", + embedding_model=embedder, reranker=reranker) + titles = corpus["title"].fillna("").astype(str) + bodies = corpus["text"].fillna("").astype(str) + texts = [(t + "\n" + b).strip() for t, b in zip(titles, bodies)] + ids = corpus["_id"].astype(str).tolist() + keep = [i for i, t in enumerate(texts) if t] + texts, ids = [texts[i] for i in keep], [ids[i] for i in keep] + step = 2000 + for start in range(0, len(texts), step): + chunk_texts, chunk_ids = texts[start:start + step], ids[start:start + step] + emb = embedder.encode(chunk_texts, batch_size=64).tolist() + engine.collection.add(ids=chunk_ids, documents=chunk_texts, embeddings=emb, + metadatas=[{"source": i} for i in chunk_ids]) + print(f" stored {start + len(chunk_texts)}/{len(texts)}", flush=True) + return engine, len(texts) + + +def sources(metas) -> list[str]: + return [m["source"] for m in metas] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--dataset", choices=sorted(DATASETS), default="scifact") + parser.add_argument("--limit", type=int, default=0, help="only the first N questions (quick check)") + args = parser.parse_args() + + corpus, questions, relevant = load(args.dataset) + if args.limit: + questions = dict(list(questions.items())[:args.limit]) + print(f"{args.dataset}: {len(corpus)} documents, {len(questions)} test questions", flush=True) + + embedder = SentenceTransformer(EMBEDDER[0], revision=EMBEDDER[1], device="cpu") + reranker = CrossEncoder(RERANKER[0], revision=RERANKER[1], device="cpu") + + with tempfile.TemporaryDirectory() as tmp: + engine, n_stored = build_engine(corpus, embedder, reranker, tmp, args.dataset) + + def dense(q): + res = engine.collection.query(query_embeddings=embedder.encode([q]).tolist(), n_results=K) + return res["ids"][0] + + def bm25(q): + return sources(engine._bm25_search(q, top_n=K)[1]) + + def hybrid_rerank(q): + return sources(engine.retrieve(q, top_k=K, confidence_threshold=0.0, max_tokens=10**9)[1]) + + bm25_search = engine._bm25_search + + def dense_rerank(q): + engine._bm25_search = lambda *_a, **_k: ([], []) + try: + return hybrid_rerank(q) + finally: + engine._bm25_search = bm25_search + + def app_default(q): + return sources(engine.retrieve(q)[1]) + + def app_no_cutoff(q): + return sources(engine.retrieve(q, top_k=APP_TOP_K, confidence_threshold=0.0)[1]) + + setups = {"dense": dense, "bm25": bm25, "dense+rerank": dense_rerank, + "hybrid+rerank": hybrid_rerank, "app_default": app_default, + "app_default_no_cutoff": app_no_cutoff} + + # Warm-up, so that one-off loading does not count in the timings. + for fn in setups.values(): + fn("warm up") + + rows = [] + for setup, fn in setups.items(): + t_all = time.perf_counter() + for qid, q in questions.items(): + t0 = time.perf_counter() + ranked = fn(q) + ms = (time.perf_counter() - t0) * 1000 + rel = relevant[qid] + rows.append({"setup": setup, "query_id": qid, "returned": len(ranked), "ms": ms, + "ndcg10": ndcg_at_k(ranked, rel, K), "recall10": recall_at_k(ranked, rel, K), + "mrr10": mrr_at_k(ranked, rel, K), "hit3": hit_at_k(ranked, rel, 3)}) + print(f" {setup}: {time.perf_counter() - t_all:.0f} s", flush=True) + + per_q = pd.DataFrame(rows) + summary = {} + for setup, g in per_q.groupby("setup", sort=False): + summary[setup] = { + "ndcg@10": round(g["ndcg10"].mean(), 4), + "recall@10": round(g["recall10"].mean(), 4), + "mrr@10": round(g["mrr10"].mean(), 4), + "hit@3": round(g["hit3"].mean(), 4), + "no_passage_share": round(float((g["returned"] == 0).mean()), 4), + "ms_per_question_median": round(statistics.median(g["ms"]), 1), + } + + result = { + "dataset": args.dataset, + "documents_in_corpus": len(corpus), + "documents_stored": n_stored, + "test_questions": len(questions), + "full_test_set": not args.limit, + "sources": {"corpus": DATASETS[args.dataset]["corpus"], "qrels": DATASETS[args.dataset]["qrels"], + "embedder": EMBEDDER, "reranker": RERANKER}, + "machine": f"{platform.machine()}, {platform.system()} {platform.release()}, CPU only", + "packages": {p: version(p) for p in ["chromadb", "sentence-transformers", "torch", "rank-bm25"]}, + "setups": summary, + } + print(json.dumps(summary, indent=2)) + if args.limit: + print("Quick check only (--limit): nothing saved.") + return + all_results_path = OUT_DIR / "results.json" + all_results = json.loads(all_results_path.read_text()) if all_results_path.exists() else {} + all_results[args.dataset] = result + all_results_path.write_text(json.dumps(all_results, indent=2) + "\n") + per_q.round(5).to_csv(OUT_DIR / f"per_question_{args.dataset}.csv", index=False) + print(f"Saved {all_results_path} and per_question_{args.dataset}.csv") + + +if __name__ == "__main__": + main() diff --git a/tests/test_retrieval_metrics.py b/tests/test_retrieval_metrics.py new file mode 100644 index 0000000..c378fe6 --- /dev/null +++ b/tests/test_retrieval_metrics.py @@ -0,0 +1,59 @@ +"""Checks for the ranking metrics on small hand-made cases.""" + +import math +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from evaluation.metrics import hit_at_k, mrr_at_k, ndcg_at_k, recall_at_k # noqa: E402 + + +def test_perfect_ranking_scores_one(): + relevant = {"a": 1, "b": 1} + assert ndcg_at_k(["a", "b", "c"], relevant) == pytest.approx(1.0) + assert recall_at_k(["a", "b", "c"], relevant) == 1.0 + assert hit_at_k(["a", "b", "c"], relevant, k=1) == 1.0 + assert mrr_at_k(["a", "b", "c"], relevant) == 1.0 + + +def test_nothing_relevant_found_scores_zero(): + relevant = {"x": 1} + assert ndcg_at_k(["a", "b"], relevant) == 0.0 + assert recall_at_k(["a", "b"], relevant) == 0.0 + assert hit_at_k(["a", "b"], relevant) == 0.0 + assert mrr_at_k(["a", "b"], relevant) == 0.0 + + +def test_ndcg_one_relevant_at_rank_two(): + # DCG = 1 / log2(3); the ideal ranking puts it first, IDCG = 1. + assert ndcg_at_k(["a", "x", "b"], {"x": 1}) == pytest.approx(1 / math.log2(3)) + + +def test_ndcg_uses_grades(): + relevant = {"a": 2, "b": 1} + worse_first = ndcg_at_k(["b", "a"], relevant) + better_first = ndcg_at_k(["a", "b"], relevant) + assert better_first == pytest.approx(1.0) + assert worse_first < better_first + + +def test_cut_off_at_k(): + relevant = {"x": 1} + ranked = ["a", "b", "c", "x"] + assert hit_at_k(ranked, relevant, k=3) == 0.0 + assert hit_at_k(ranked, relevant, k=4) == 1.0 + assert recall_at_k(ranked, relevant, k=3) == 0.0 + assert mrr_at_k(ranked, relevant, k=4) == pytest.approx(0.25) + + +def test_recall_counts_share_of_relevant(): + assert recall_at_k(["a", "c"], {"a": 1, "b": 1}) == 0.5 + + +def test_zero_grade_is_not_relevant(): + relevant = {"a": 0, "b": 1} + assert hit_at_k(["a"], relevant, k=1) == 0.0 + assert recall_at_k(["a", "b"], relevant) == 1.0 From d99e728d5139fa9388ed50042946e2c4ec5ab62a Mon Sep 17 00:00:00 2001 From: YsK-dev Date: Sat, 14 Nov 2026 14:33:52 +0300 Subject: [PATCH 2/4] BM25: split text into words without punctuation Splitting on spaces kept punctuation on words, so "cancer." did not match "cancer". On SciFact this raises BM25 alone from nDCG@10 0.560 to 0.652. The full hybrid + reranker pipeline moves from 0.694 to 0.696 (no measurable change). Results re-run with the fix. --- evaluation/per_question_scifact.csv | 3600 +++++++++++++-------------- evaluation/results.json | 50 +- src/rag_engine.py | 19 +- tests/test_bm25_tokens.py | 24 + 4 files changed, 1861 insertions(+), 1832 deletions(-) create mode 100644 tests/test_bm25_tokens.py diff --git a/evaluation/per_question_scifact.csv b/evaluation/per_question_scifact.csv index 4a45ab1..68368f0 100644 --- a/evaluation/per_question_scifact.csv +++ b/evaluation/per_question_scifact.csv @@ -1,1801 +1,1801 @@ setup,query_id,returned,ms,ndcg10,recall10,mrr10,hit3 -dense,1,10,9.68054,0.38685,1.0,0.2,0.0 -dense,3,10,9.9485,0.5,1.0,0.33333,1.0 -dense,5,10,9.0325,0.63093,1.0,0.5,1.0 -dense,13,10,9.73871,0.0,0.0,0.0,0.0 -dense,36,10,11.26167,0.49115,1.0,0.33333,1.0 -dense,42,10,9.91908,0.63093,1.0,0.5,1.0 -dense,48,10,8.64708,0.0,0.0,0.0,0.0 -dense,49,10,9.86017,1.0,1.0,1.0,1.0 -dense,50,10,9.69054,1.0,1.0,1.0,1.0 -dense,51,10,9.57742,1.0,1.0,1.0,1.0 -dense,53,10,9.72192,1.0,1.0,1.0,1.0 -dense,54,10,9.63629,1.0,1.0,1.0,1.0 -dense,56,10,10.07879,1.0,1.0,1.0,1.0 -dense,57,10,9.3725,1.0,1.0,1.0,1.0 -dense,70,10,9.55012,0.0,0.0,0.0,0.0 -dense,72,10,9.80296,1.0,1.0,1.0,1.0 -dense,75,10,9.55596,0.63093,1.0,0.5,1.0 -dense,94,10,8.96471,0.5,1.0,0.33333,1.0 -dense,99,10,9.29692,0.63093,1.0,0.5,1.0 -dense,100,10,9.58583,0.5,1.0,0.33333,1.0 -dense,113,10,9.50358,1.0,1.0,1.0,1.0 -dense,115,10,9.27246,1.0,1.0,1.0,1.0 -dense,118,10,11.43742,0.63093,1.0,0.5,1.0 -dense,124,10,10.30162,1.0,1.0,1.0,1.0 -dense,127,10,9.63,1.0,1.0,1.0,1.0 -dense,128,10,9.60729,0.0,0.0,0.0,0.0 -dense,129,10,8.95175,1.0,1.0,1.0,1.0 -dense,130,10,9.49037,1.0,1.0,1.0,1.0 -dense,132,10,9.54454,0.0,0.0,0.0,0.0 -dense,133,10,9.61846,0.63995,0.6,1.0,1.0 -dense,137,10,9.51329,1.0,1.0,1.0,1.0 -dense,141,10,9.42083,0.91972,1.0,1.0,1.0 -dense,142,10,9.86571,1.0,1.0,1.0,1.0 -dense,143,10,9.44371,1.0,1.0,1.0,1.0 -dense,146,10,11.00546,1.0,1.0,1.0,1.0 -dense,148,10,8.79483,0.63093,1.0,0.5,1.0 -dense,163,10,9.50354,1.0,1.0,1.0,1.0 -dense,171,10,9.84758,1.0,1.0,1.0,1.0 -dense,179,10,8.99012,0.95583,1.0,1.0,1.0 -dense,180,10,10.50846,1.0,1.0,1.0,1.0 -dense,183,10,8.53538,1.0,1.0,1.0,1.0 -dense,185,10,9.06525,1.0,1.0,1.0,1.0 -dense,198,10,9.37179,0.0,0.0,0.0,0.0 -dense,208,10,9.18637,0.5,1.0,0.33333,1.0 -dense,212,10,8.85512,0.35621,1.0,0.16667,0.0 -dense,213,10,10.33663,0.38685,1.0,0.2,0.0 -dense,216,10,9.03325,1.0,1.0,1.0,1.0 -dense,217,10,9.53658,1.0,1.0,1.0,1.0 -dense,218,10,9.21404,1.0,1.0,1.0,1.0 -dense,219,10,9.3155,1.0,1.0,1.0,1.0 -dense,230,10,9.10767,1.0,1.0,1.0,1.0 -dense,232,10,9.0945,1.0,1.0,1.0,1.0 -dense,233,10,9.04679,1.0,1.0,1.0,1.0 -dense,236,10,8.77438,1.0,1.0,1.0,1.0 -dense,237,10,9.10054,1.0,1.0,1.0,1.0 -dense,238,10,9.97,0.63093,1.0,0.5,1.0 -dense,239,10,9.04325,0.0,0.0,0.0,0.0 -dense,248,10,9.58875,0.5,1.0,0.33333,1.0 -dense,249,10,9.70213,0.5,1.0,0.33333,1.0 -dense,261,10,11.02521,0.20438,0.5,0.14286,0.0 -dense,268,10,8.12212,1.0,1.0,1.0,1.0 -dense,269,10,8.17967,1.0,1.0,1.0,1.0 -dense,274,10,10.61404,1.0,1.0,1.0,1.0 -dense,275,10,9.52712,0.67973,1.0,0.5,1.0 -dense,279,10,9.00908,1.0,1.0,1.0,1.0 -dense,294,10,9.40754,0.0,0.0,0.0,0.0 -dense,295,10,9.14608,0.5,1.0,0.33333,1.0 -dense,298,10,8.93442,1.0,1.0,1.0,1.0 -dense,300,10,9.91088,0.43068,1.0,0.25,0.0 -dense,303,10,8.95063,0.0,0.0,0.0,0.0 -dense,312,10,10.54762,0.0,0.0,0.0,0.0 -dense,314,10,10.12454,0.0,0.0,0.0,0.0 -dense,324,10,9.10613,0.63093,1.0,0.5,1.0 -dense,327,10,10.08621,1.0,1.0,1.0,1.0 -dense,338,10,10.25475,0.63093,1.0,0.5,1.0 -dense,343,10,10.12292,0.83155,1.0,1.0,1.0 -dense,350,10,8.83946,1.0,1.0,1.0,1.0 -dense,354,10,8.95508,1.0,1.0,1.0,1.0 -dense,362,10,10.29908,1.0,1.0,1.0,1.0 -dense,380,10,8.84217,1.0,1.0,1.0,1.0 -dense,384,10,8.97638,0.0,0.0,0.0,0.0 -dense,385,10,9.00012,0.69343,1.0,0.5,1.0 -dense,386,10,10.02521,1.0,1.0,1.0,1.0 -dense,388,10,8.70112,0.63093,1.0,0.5,1.0 -dense,399,10,9.97079,1.0,1.0,1.0,1.0 -dense,410,10,9.88042,1.0,1.0,1.0,1.0 -dense,411,10,8.96533,1.0,1.0,1.0,1.0 -dense,415,10,9.35758,0.0,0.0,0.0,0.0 -dense,421,10,8.98279,0.0,0.0,0.0,0.0 -dense,431,10,9.69217,0.0,0.0,0.0,0.0 -dense,436,10,8.78946,1.0,1.0,1.0,1.0 -dense,437,10,8.67092,0.0,0.0,0.0,0.0 -dense,439,10,8.853,1.0,1.0,1.0,1.0 -dense,440,10,9.07258,1.0,1.0,1.0,1.0 -dense,443,10,9.17042,1.0,1.0,1.0,1.0 -dense,452,10,10.84625,0.17724,0.5,0.1,0.0 -dense,475,10,9.43958,0.0,0.0,0.0,0.0 -dense,478,10,10.48404,1.0,1.0,1.0,1.0 -dense,491,10,8.8025,1.0,1.0,1.0,1.0 -dense,501,10,10.09521,1.0,1.0,1.0,1.0 -dense,502,10,9.10092,0.0,0.0,0.0,0.0 -dense,507,10,8.86829,0.63093,1.0,0.5,1.0 -dense,508,10,8.9325,0.0,0.0,0.0,0.0 -dense,513,10,9.94712,0.63093,1.0,0.5,1.0 -dense,514,10,9.544,1.0,1.0,1.0,1.0 -dense,516,10,10.38388,1.0,1.0,1.0,1.0 -dense,517,10,9.22208,0.0,0.0,0.0,0.0 -dense,521,10,9.91183,1.0,1.0,1.0,1.0 -dense,525,10,9.40804,1.0,1.0,1.0,1.0 -dense,527,10,10.47533,0.0,0.0,0.0,0.0 -dense,528,10,10.70675,1.0,1.0,1.0,1.0 -dense,532,10,9.34075,1.0,1.0,1.0,1.0 -dense,533,10,9.31912,1.0,1.0,1.0,1.0 -dense,535,10,9.38425,0.43068,1.0,0.25,0.0 -dense,536,10,9.14192,1.0,1.0,1.0,1.0 -dense,539,10,9.63604,1.0,1.0,1.0,1.0 -dense,540,10,10.00492,0.26407,0.5,0.25,0.0 -dense,544,10,9.15292,0.0,0.0,0.0,0.0 -dense,549,10,9.15188,1.0,1.0,1.0,1.0 -dense,551,10,9.70779,0.63093,1.0,0.5,1.0 -dense,552,10,9.8205,1.0,1.0,1.0,1.0 -dense,554,10,9.47238,0.0,0.0,0.0,0.0 -dense,560,10,9.20363,0.0,0.0,0.0,0.0 -dense,569,10,9.33037,0.43068,1.0,0.25,0.0 -dense,575,10,9.2365,0.5,1.0,0.33333,1.0 -dense,577,10,10.27333,0.31546,1.0,0.125,0.0 -dense,578,10,9.38229,1.0,1.0,1.0,1.0 -dense,587,10,10.41129,0.0,0.0,0.0,0.0 -dense,589,10,9.51662,1.0,1.0,1.0,1.0 -dense,593,10,9.69929,1.0,1.0,1.0,1.0 -dense,597,10,9.92012,0.79287,1.0,1.0,1.0 -dense,598,10,9.49717,0.33333,1.0,0.14286,0.0 -dense,613,10,9.19129,1.0,1.0,1.0,1.0 -dense,619,10,9.20262,0.0,0.0,0.0,0.0 -dense,623,10,9.97762,0.0,0.0,0.0,0.0 -dense,628,10,9.45025,0.43068,1.0,0.25,0.0 -dense,636,10,10.62654,1.0,1.0,1.0,1.0 -dense,637,10,8.84067,0.43068,1.0,0.25,0.0 -dense,641,10,9.35792,0.83155,1.0,1.0,1.0 -dense,644,10,9.0545,0.0,0.0,0.0,0.0 -dense,649,10,9.27513,1.0,1.0,1.0,1.0 -dense,659,10,10.28037,0.43068,1.0,0.25,0.0 -dense,660,10,9.48775,0.0,0.0,0.0,0.0 -dense,674,10,9.29629,0.33333,1.0,0.14286,0.0 -dense,684,10,8.78875,1.0,1.0,1.0,1.0 -dense,690,10,9.57229,0.0,0.0,0.0,0.0 -dense,691,10,9.221,0.43068,1.0,0.25,0.0 -dense,692,10,9.23163,1.0,1.0,1.0,1.0 -dense,693,10,9.39579,1.0,1.0,1.0,1.0 -dense,700,10,9.13404,0.63093,1.0,0.5,1.0 -dense,702,10,8.82663,0.63093,1.0,0.5,1.0 -dense,715,10,8.82558,0.0,0.0,0.0,0.0 -dense,716,10,10.54804,0.0,0.0,0.0,0.0 -dense,718,10,9.31846,1.0,1.0,1.0,1.0 -dense,721,10,9.39229,1.0,1.0,1.0,1.0 -dense,723,10,9.67475,1.0,1.0,1.0,1.0 -dense,727,10,9.16004,0.43068,1.0,0.25,0.0 -dense,728,10,9.12346,0.65092,1.0,0.5,1.0 -dense,729,10,8.94025,1.0,1.0,1.0,1.0 -dense,742,10,9.33454,0.30103,1.0,0.11111,0.0 -dense,743,10,9.08408,0.0,0.0,0.0,0.0 -dense,744,10,9.26138,0.5,1.0,0.33333,1.0 -dense,756,10,8.833,0.63093,1.0,0.5,1.0 -dense,759,10,10.05567,1.0,1.0,1.0,1.0 -dense,768,10,10.24592,0.0,0.0,0.0,0.0 -dense,770,10,10.33617,1.0,1.0,1.0,1.0 -dense,775,10,9.57192,0.0,0.0,0.0,0.0 -dense,781,10,9.043,0.63093,1.0,0.5,1.0 -dense,783,10,9.39675,0.38685,1.0,0.2,0.0 -dense,784,10,8.95208,0.63093,1.0,0.5,1.0 -dense,785,10,8.6995,0.0,0.0,0.0,0.0 -dense,793,10,10.12779,1.0,1.0,1.0,1.0 -dense,800,10,9.748,0.63093,1.0,0.5,1.0 -dense,805,10,10.29742,1.0,1.0,1.0,1.0 -dense,808,10,9.44617,1.0,1.0,1.0,1.0 -dense,811,10,9.04804,1.0,1.0,1.0,1.0 -dense,814,10,11.33271,1.0,1.0,1.0,1.0 -dense,820,10,9.2015,0.0,0.0,0.0,0.0 -dense,821,10,9.38367,0.0,0.0,0.0,0.0 -dense,823,10,9.40279,1.0,1.0,1.0,1.0 -dense,830,10,10.61463,0.0,0.0,0.0,0.0 -dense,831,10,9.62538,0.0,0.0,0.0,0.0 -dense,832,10,9.34396,1.0,1.0,1.0,1.0 -dense,834,10,9.33867,0.0,0.0,0.0,0.0 -dense,837,10,9.7265,0.63093,1.0,0.5,1.0 -dense,839,10,13.53763,0.5,1.0,0.33333,1.0 -dense,845,10,10.75008,0.43068,1.0,0.25,0.0 -dense,847,10,11.44754,1.0,1.0,1.0,1.0 -dense,852,10,9.458,0.63093,1.0,0.5,1.0 -dense,859,10,10.22104,0.43068,1.0,0.25,0.0 -dense,870,10,9.05288,0.0,0.0,0.0,0.0 -dense,873,10,9.64763,0.26762,0.4,0.33333,1.0 -dense,879,10,9.59429,1.0,1.0,1.0,1.0 -dense,880,10,9.73196,0.63093,1.0,0.5,1.0 -dense,882,10,8.99317,1.0,1.0,1.0,1.0 -dense,887,10,9.29692,0.0,0.0,0.0,0.0 -dense,903,10,9.51708,1.0,1.0,1.0,1.0 -dense,904,10,10.32637,1.0,1.0,1.0,1.0 -dense,907,10,9.85862,1.0,1.0,1.0,1.0 -dense,911,10,9.03888,1.0,1.0,1.0,1.0 -dense,913,10,11.14817,0.31546,1.0,0.125,0.0 -dense,914,10,9.95054,0.43068,1.0,0.25,0.0 -dense,921,10,10.45258,1.0,1.0,1.0,1.0 -dense,922,10,11.337,1.0,1.0,1.0,1.0 -dense,936,10,12.71867,1.0,1.0,1.0,1.0 -dense,956,10,14.41079,1.0,1.0,1.0,1.0 -dense,957,10,11.19117,0.5,1.0,0.33333,1.0 -dense,960,10,9.11888,1.0,1.0,1.0,1.0 -dense,967,10,10.498,1.0,1.0,1.0,1.0 -dense,971,10,9.48108,0.73157,1.0,0.5,1.0 -dense,975,10,10.276,0.0,0.0,0.0,0.0 -dense,982,10,9.26346,1.0,1.0,1.0,1.0 -dense,985,10,9.19404,1.0,1.0,1.0,1.0 -dense,993,10,9.85346,0.63093,1.0,0.5,1.0 -dense,1012,10,9.73392,1.0,1.0,1.0,1.0 -dense,1014,10,10.69617,1.0,1.0,1.0,1.0 -dense,1019,10,10.93546,0.43068,1.0,0.25,0.0 -dense,1020,10,9.62887,0.63093,1.0,0.5,1.0 -dense,1021,10,9.44629,0.63093,1.0,0.5,1.0 -dense,1024,10,11.47979,0.0,0.0,0.0,0.0 -dense,1029,10,10.41608,0.96747,1.0,1.0,1.0 -dense,1041,10,10.34496,0.0,0.0,0.0,0.0 -dense,1049,10,10.06758,0.0,0.0,0.0,0.0 -dense,1062,10,10.55487,0.63093,1.0,0.5,1.0 -dense,1086,10,10.00437,1.0,1.0,1.0,1.0 -dense,1088,10,10.55429,1.0,1.0,1.0,1.0 -dense,1089,10,9.49404,1.0,1.0,1.0,1.0 -dense,1099,10,8.83654,0.5,1.0,0.33333,1.0 -dense,1100,10,8.27146,0.43068,1.0,0.25,0.0 -dense,1104,10,10.27875,1.0,1.0,1.0,1.0 -dense,1107,10,10.08379,0.5,1.0,0.33333,1.0 -dense,1110,10,9.19833,0.0,0.0,0.0,0.0 -dense,1121,10,8.96383,1.0,1.0,1.0,1.0 -dense,1130,10,9.69629,1.0,1.0,1.0,1.0 -dense,1132,10,8.97925,0.2372,0.5,0.2,0.0 -dense,1137,10,9.61417,1.0,1.0,1.0,1.0 -dense,1140,10,9.05938,1.0,1.0,1.0,1.0 -dense,1144,10,9.41842,1.0,1.0,1.0,1.0 -dense,1146,10,9.73996,1.0,1.0,1.0,1.0 -dense,1150,10,9.16467,1.0,1.0,1.0,1.0 -dense,1163,10,8.88825,1.0,1.0,1.0,1.0 -dense,1175,10,8.99779,0.0,0.0,0.0,0.0 -dense,1179,10,9.07746,0.0,0.0,0.0,0.0 -dense,1180,10,8.65729,0.0,0.0,0.0,0.0 -dense,1185,10,11.13533,0.5,1.0,0.33333,1.0 -dense,1187,10,9.71167,0.63093,1.0,0.5,1.0 -dense,1191,10,8.96042,0.63093,1.0,0.5,1.0 -dense,1194,10,9.35987,1.0,1.0,1.0,1.0 -dense,1196,10,9.54392,0.38685,1.0,0.2,0.0 -dense,1197,10,9.57471,0.38685,1.0,0.2,0.0 -dense,1199,10,9.10854,0.0,0.0,0.0,0.0 -dense,1200,10,10.15933,0.0,0.0,0.0,0.0 -dense,1202,10,9.11096,0.63093,1.0,0.5,1.0 -dense,1204,10,9.69817,1.0,1.0,1.0,1.0 -dense,1207,10,10.71412,1.0,1.0,1.0,1.0 -dense,1213,10,9.67179,0.0,0.0,0.0,0.0 -dense,1216,10,9.92546,1.0,1.0,1.0,1.0 -dense,1221,10,9.09579,0.31546,1.0,0.125,0.0 -dense,1225,10,9.72125,1.0,1.0,1.0,1.0 -dense,1226,10,10.08121,0.0,0.0,0.0,0.0 -dense,1232,10,8.90667,1.0,1.0,1.0,1.0 -dense,1241,10,8.74358,1.0,1.0,1.0,1.0 -dense,1245,10,9.88354,0.30103,1.0,0.11111,0.0 -dense,1259,10,10.11171,0.63093,1.0,0.5,1.0 -dense,1262,10,9.00242,1.0,1.0,1.0,1.0 -dense,1266,10,10.32563,0.38685,1.0,0.2,0.0 -dense,1270,10,9.13229,1.0,1.0,1.0,1.0 -dense,1271,10,9.87979,1.0,1.0,1.0,1.0 -dense,1272,10,9.02512,0.0,0.0,0.0,0.0 -dense,1273,10,9.14375,1.0,1.0,1.0,1.0 -dense,1274,10,9.65025,0.9469,1.0,1.0,1.0 -dense,1278,10,9.49913,1.0,1.0,1.0,1.0 -dense,1279,10,8.79987,1.0,1.0,1.0,1.0 -dense,1280,10,9.60288,0.0,0.0,0.0,0.0 -dense,1281,10,9.28608,0.31546,1.0,0.125,0.0 -dense,1282,10,9.50971,1.0,1.0,1.0,1.0 -dense,1290,10,9.42429,1.0,1.0,1.0,1.0 -dense,1292,10,9.67167,1.0,1.0,1.0,1.0 -dense,1298,10,9.51771,1.0,1.0,1.0,1.0 -dense,1303,10,9.60367,1.0,1.0,1.0,1.0 -dense,1316,10,9.76221,0.0,0.0,0.0,0.0 -dense,1319,10,9.31358,1.0,1.0,1.0,1.0 -dense,1320,10,8.95312,1.0,1.0,1.0,1.0 -dense,1332,10,9.83483,0.0,0.0,0.0,0.0 -dense,1335,10,9.21542,1.0,1.0,1.0,1.0 -dense,1336,10,10.11729,1.0,1.0,1.0,1.0 -dense,1337,10,9.89596,0.28906,1.0,0.1,0.0 -dense,1339,10,9.29675,1.0,1.0,1.0,1.0 -dense,1344,10,9.75092,0.63093,1.0,0.5,1.0 -dense,1352,10,9.35767,1.0,1.0,1.0,1.0 -dense,1359,10,9.50817,1.0,1.0,1.0,1.0 -dense,1362,10,8.99062,0.0,0.0,0.0,0.0 -dense,1363,10,9.18075,0.0,0.0,0.0,0.0 -dense,1368,10,9.34963,0.0,0.0,0.0,0.0 -dense,1370,10,8.71604,0.5,1.0,0.33333,1.0 -dense,1379,10,9.36408,0.95583,1.0,1.0,1.0 -dense,1382,10,9.42504,1.0,1.0,1.0,1.0 -dense,1385,10,9.00183,0.63093,1.0,0.5,1.0 -dense,1389,10,9.63775,1.0,1.0,1.0,1.0 -dense,1395,10,10.01796,0.5,1.0,0.33333,1.0 -bm25,1,10,5.02104,0.0,0.0,0.0,0.0 -bm25,3,10,16.10962,0.63093,1.0,0.5,1.0 -bm25,5,10,6.61529,1.0,1.0,1.0,1.0 -bm25,13,10,9.02621,0.0,0.0,0.0,0.0 -bm25,36,10,10.14104,0.0,0.0,0.0,0.0 -bm25,42,10,12.69625,1.0,1.0,1.0,1.0 -bm25,48,10,12.12929,0.5,1.0,0.33333,1.0 -bm25,49,10,5.87804,1.0,1.0,1.0,1.0 -bm25,50,10,7.06804,1.0,1.0,1.0,1.0 -bm25,51,10,8.85283,1.0,1.0,1.0,1.0 -bm25,53,10,9.27504,1.0,1.0,1.0,1.0 -bm25,54,10,10.36696,0.0,0.0,0.0,0.0 -bm25,56,10,11.88525,1.0,1.0,1.0,1.0 -bm25,57,10,12.41467,1.0,1.0,1.0,1.0 -bm25,70,10,6.15571,1.0,1.0,1.0,1.0 -bm25,72,10,6.86937,0.5,1.0,0.33333,1.0 -bm25,75,10,12.24871,0.63093,1.0,0.5,1.0 -bm25,94,10,6.72263,0.0,0.0,0.0,0.0 -bm25,99,10,9.47204,1.0,1.0,1.0,1.0 -bm25,100,10,7.18354,1.0,1.0,1.0,1.0 -bm25,113,10,12.35,0.63093,1.0,0.5,1.0 -bm25,115,10,9.70908,0.63093,1.0,0.5,1.0 -bm25,118,10,10.08258,1.0,1.0,1.0,1.0 -bm25,124,10,11.41567,1.0,1.0,1.0,1.0 -bm25,127,10,11.41371,0.0,0.0,0.0,0.0 -bm25,128,10,7.05846,0.0,0.0,0.0,0.0 -bm25,129,10,13.03096,1.0,1.0,1.0,1.0 -bm25,130,10,13.70837,1.0,1.0,1.0,1.0 -bm25,132,10,5.46787,0.0,0.0,0.0,0.0 -bm25,133,10,15.60187,0.32095,0.6,0.16667,0.0 -bm25,137,10,10.91925,1.0,1.0,1.0,1.0 -bm25,141,10,9.63342,0.91972,1.0,1.0,1.0 -bm25,142,10,17.07158,1.0,1.0,1.0,1.0 -bm25,143,10,16.56092,1.0,1.0,1.0,1.0 -bm25,146,10,15.92083,1.0,1.0,1.0,1.0 -bm25,148,10,6.59904,1.0,1.0,1.0,1.0 -bm25,163,10,10.38608,0.63093,1.0,0.5,1.0 -bm25,171,10,11.39996,1.0,1.0,1.0,1.0 -bm25,179,10,7.01017,0.12315,0.25,0.125,0.0 -bm25,180,10,16.18529,1.0,1.0,1.0,1.0 -bm25,183,10,7.38754,0.38685,1.0,0.2,0.0 -bm25,185,10,8.93542,0.43068,1.0,0.25,0.0 -bm25,198,10,5.07875,0.0,0.0,0.0,0.0 -bm25,208,10,6.92583,1.0,1.0,1.0,1.0 -bm25,212,10,7.00362,1.0,1.0,1.0,1.0 -bm25,213,10,11.82817,0.38685,1.0,0.2,0.0 -bm25,216,10,10.95425,1.0,1.0,1.0,1.0 -bm25,217,10,10.43204,1.0,1.0,1.0,1.0 -bm25,218,10,9.41396,1.0,1.0,1.0,1.0 -bm25,219,10,9.50104,1.0,1.0,1.0,1.0 -bm25,230,10,14.27471,1.0,1.0,1.0,1.0 -bm25,232,10,14.42071,0.63093,1.0,0.5,1.0 -bm25,233,10,13.89271,0.5,1.0,0.33333,1.0 -bm25,236,10,10.35388,1.0,1.0,1.0,1.0 -bm25,237,10,11.11283,0.5,1.0,0.33333,1.0 -bm25,238,10,7.34821,0.0,0.0,0.0,0.0 -bm25,239,10,8.67125,0.0,0.0,0.0,0.0 -bm25,248,10,9.62742,1.0,1.0,1.0,1.0 -bm25,249,10,6.99225,1.0,1.0,1.0,1.0 -bm25,261,10,11.54746,0.30657,0.5,0.33333,1.0 -bm25,268,10,5.35275,0.43068,1.0,0.25,0.0 -bm25,269,10,6.19833,0.5,1.0,0.33333,1.0 -bm25,274,10,20.68396,1.0,1.0,1.0,1.0 -bm25,275,10,14.4565,0.46928,0.33333,1.0,1.0 -bm25,279,10,11.52517,1.0,1.0,1.0,1.0 -bm25,294,10,13.74958,0.0,0.0,0.0,0.0 -bm25,295,10,23.51008,1.0,1.0,1.0,1.0 -bm25,298,10,18.417,1.0,1.0,1.0,1.0 -bm25,300,10,25.592,0.0,0.0,0.0,0.0 -bm25,303,10,12.64154,0.43068,1.0,0.25,0.0 -bm25,312,10,11.51921,0.30103,1.0,0.11111,0.0 -bm25,314,10,16.93296,0.5,1.0,0.33333,1.0 -bm25,324,10,4.30133,0.0,0.0,0.0,0.0 -bm25,327,10,9.28308,1.0,1.0,1.0,1.0 -bm25,338,10,5.64962,1.0,1.0,1.0,1.0 -bm25,343,10,14.01117,0.57064,1.0,0.33333,1.0 -bm25,350,10,11.4735,0.63093,1.0,0.5,1.0 -bm25,354,10,9.56746,1.0,1.0,1.0,1.0 -bm25,362,10,19.46746,1.0,1.0,1.0,1.0 -bm25,380,10,10.73308,1.0,1.0,1.0,1.0 -bm25,384,10,11.15062,0.0,0.0,0.0,0.0 -bm25,385,10,10.46054,0.62405,1.0,0.5,1.0 -bm25,386,10,13.07983,1.0,1.0,1.0,1.0 -bm25,388,10,8.69504,0.5,1.0,0.33333,1.0 -bm25,399,10,9.95933,1.0,1.0,1.0,1.0 -bm25,410,10,9.42967,1.0,1.0,1.0,1.0 -bm25,411,10,8.97554,1.0,1.0,1.0,1.0 -bm25,415,10,10.32767,0.31546,1.0,0.125,0.0 -bm25,421,10,10.32025,0.0,0.0,0.0,0.0 -bm25,431,10,10.73333,0.0,0.0,0.0,0.0 -bm25,436,10,10.99133,1.0,1.0,1.0,1.0 -bm25,437,10,16.66796,0.0,0.0,0.0,0.0 -bm25,439,10,12.00158,0.0,0.0,0.0,0.0 -bm25,440,10,10.99375,0.0,0.0,0.0,0.0 -bm25,443,10,8.30275,1.0,1.0,1.0,1.0 -bm25,452,10,8.78629,0.19343,0.5,0.125,0.0 -bm25,475,10,10.08129,0.33333,1.0,0.14286,0.0 -bm25,478,10,20.49008,0.0,0.0,0.0,0.0 -bm25,491,10,11.53675,1.0,1.0,1.0,1.0 -bm25,501,10,7.27008,0.33333,1.0,0.14286,0.0 -bm25,502,10,13.50767,0.0,0.0,0.0,0.0 -bm25,507,10,12.21292,0.63093,1.0,0.5,1.0 -bm25,508,10,9.83792,0.0,0.0,0.0,0.0 -bm25,513,10,6.93387,0.63093,1.0,0.5,1.0 -bm25,514,10,15.93187,0.0,0.0,0.0,0.0 -bm25,516,10,11.90146,1.0,1.0,1.0,1.0 -bm25,517,10,6.40633,0.0,0.0,0.0,0.0 -bm25,521,10,19.97596,1.0,1.0,1.0,1.0 -bm25,525,10,16.79567,1.0,1.0,1.0,1.0 -bm25,527,10,13.79612,0.63093,1.0,0.5,1.0 -bm25,528,10,18.75946,1.0,1.0,1.0,1.0 -bm25,532,10,5.94421,1.0,1.0,1.0,1.0 -bm25,533,10,5.55604,1.0,1.0,1.0,1.0 -bm25,535,10,8.53521,0.0,0.0,0.0,0.0 -bm25,536,10,5.86271,0.0,0.0,0.0,0.0 -bm25,539,10,5.88562,1.0,1.0,1.0,1.0 -bm25,540,10,7.56342,0.38685,0.5,0.5,1.0 -bm25,544,10,8.02296,0.0,0.0,0.0,0.0 -bm25,549,10,6.46258,1.0,1.0,1.0,1.0 -bm25,551,10,21.93738,0.35621,1.0,0.16667,0.0 -bm25,552,10,17.50737,1.0,1.0,1.0,1.0 -bm25,554,10,12.31558,0.38685,1.0,0.2,0.0 -bm25,560,10,11.38021,0.0,0.0,0.0,0.0 -bm25,569,10,9.60021,1.0,1.0,1.0,1.0 -bm25,575,10,14.54058,0.0,0.0,0.0,0.0 -bm25,577,10,21.57025,0.0,0.0,0.0,0.0 -bm25,578,10,8.77196,1.0,1.0,1.0,1.0 -bm25,587,10,24.99725,0.0,0.0,0.0,0.0 -bm25,589,10,17.98575,1.0,1.0,1.0,1.0 -bm25,593,10,10.17729,1.0,1.0,1.0,1.0 -bm25,597,10,7.14267,0.88546,1.0,1.0,1.0 -bm25,598,10,19.60375,0.38685,1.0,0.2,0.0 -bm25,613,10,11.60567,1.0,1.0,1.0,1.0 -bm25,619,10,13.19783,0.26407,0.5,0.25,0.0 -bm25,623,10,10.81379,0.0,0.0,0.0,0.0 -bm25,628,10,13.15508,0.33333,1.0,0.14286,0.0 -bm25,636,10,7.91179,0.38685,1.0,0.2,0.0 -bm25,637,10,11.4995,0.33333,1.0,0.14286,0.0 -bm25,641,10,8.549,0.61315,0.5,1.0,1.0 -bm25,644,10,6.16375,0.63093,1.0,0.5,1.0 -bm25,649,10,10.29242,0.33333,1.0,0.14286,0.0 -bm25,659,10,6.43842,0.0,0.0,0.0,0.0 -bm25,660,10,5.22396,0.0,0.0,0.0,0.0 -bm25,674,10,10.51621,1.0,1.0,1.0,1.0 -bm25,684,10,9.70437,0.63093,1.0,0.5,1.0 -bm25,690,10,17.32883,0.0,0.0,0.0,0.0 -bm25,691,10,10.95983,1.0,1.0,1.0,1.0 -bm25,692,10,9.01617,0.31546,1.0,0.125,0.0 -bm25,693,10,8.58704,0.33333,1.0,0.14286,0.0 -bm25,700,10,9.78167,0.63093,1.0,0.5,1.0 -bm25,702,10,9.84279,0.31546,1.0,0.125,0.0 -bm25,715,10,14.15738,0.0,0.0,0.0,0.0 -bm25,716,10,10.13142,0.0,0.0,0.0,0.0 -bm25,718,10,8.77058,0.33333,1.0,0.14286,0.0 -bm25,721,10,11.01337,1.0,1.0,1.0,1.0 -bm25,723,10,12.66137,1.0,1.0,1.0,1.0 -bm25,727,10,11.39992,0.0,0.0,0.0,0.0 -bm25,728,10,10.09354,0.80657,1.0,1.0,1.0 -bm25,729,10,9.97425,0.63093,1.0,0.5,1.0 -bm25,742,10,6.49692,1.0,1.0,1.0,1.0 -bm25,743,10,3.5305,1.0,1.0,1.0,1.0 -bm25,744,10,13.04967,0.63093,1.0,0.5,1.0 -bm25,756,10,13.20254,0.0,0.0,0.0,0.0 -bm25,759,10,15.40096,1.0,1.0,1.0,1.0 -bm25,768,10,9.79317,0.0,0.0,0.0,0.0 -bm25,770,10,23.25229,1.0,1.0,1.0,1.0 -bm25,775,10,13.56971,0.0,0.0,0.0,0.0 -bm25,781,10,11.92079,0.43068,1.0,0.25,0.0 -bm25,783,10,11.67925,0.0,0.0,0.0,0.0 -bm25,784,10,13.35312,0.63093,1.0,0.5,1.0 -bm25,785,10,14.35275,0.63093,1.0,0.5,1.0 -bm25,793,10,5.24975,1.0,1.0,1.0,1.0 -bm25,800,10,15.33183,0.0,0.0,0.0,0.0 -bm25,805,10,6.11533,1.0,1.0,1.0,1.0 -bm25,808,10,8.36138,0.63093,1.0,0.5,1.0 -bm25,811,10,11.984,1.0,1.0,1.0,1.0 -bm25,814,10,19.30812,1.0,1.0,1.0,1.0 -bm25,820,10,6.68796,0.0,0.0,0.0,0.0 -bm25,821,10,6.40971,0.0,0.0,0.0,0.0 -bm25,823,10,6.27604,1.0,1.0,1.0,1.0 -bm25,830,10,13.25783,0.0,0.0,0.0,0.0 -bm25,831,10,10.09417,0.0,0.0,0.0,0.0 -bm25,832,10,5.34204,1.0,1.0,1.0,1.0 -bm25,834,10,9.67038,0.33333,1.0,0.14286,0.0 -bm25,837,10,7.47521,1.0,1.0,1.0,1.0 -bm25,839,10,11.86112,0.5,1.0,0.33333,1.0 -bm25,845,10,7.67575,1.0,1.0,1.0,1.0 -bm25,847,10,14.48383,1.0,1.0,1.0,1.0 -bm25,852,10,12.94467,0.63093,1.0,0.5,1.0 -bm25,859,10,6.75488,0.63093,1.0,0.5,1.0 -bm25,870,10,3.3905,0.0,0.0,0.0,0.0 -bm25,873,10,7.05704,0.0,0.0,0.0,0.0 -bm25,879,10,8.94933,1.0,1.0,1.0,1.0 -bm25,880,10,7.33754,1.0,1.0,1.0,1.0 -bm25,882,10,7.823,0.63093,1.0,0.5,1.0 -bm25,887,10,12.15438,0.0,0.0,0.0,0.0 -bm25,903,10,7.97412,1.0,1.0,1.0,1.0 -bm25,904,10,17.32929,1.0,1.0,1.0,1.0 -bm25,907,10,14.08487,1.0,1.0,1.0,1.0 -bm25,911,10,11.55475,1.0,1.0,1.0,1.0 -bm25,913,10,5.93296,0.0,0.0,0.0,0.0 -bm25,914,10,6.07092,0.0,0.0,0.0,0.0 -bm25,921,10,8.10558,0.43068,1.0,0.25,0.0 -bm25,922,10,10.38783,1.0,1.0,1.0,1.0 -bm25,936,10,6.84962,1.0,1.0,1.0,1.0 -bm25,956,10,11.36442,0.63093,1.0,0.5,1.0 -bm25,957,10,9.76367,0.5,1.0,0.33333,1.0 -bm25,960,10,4.39825,1.0,1.0,1.0,1.0 -bm25,967,10,7.89654,0.69343,1.0,0.5,1.0 -bm25,971,10,16.97829,0.64446,1.0,0.33333,1.0 -bm25,975,10,7.50396,0.0,0.0,0.0,0.0 -bm25,982,10,15.5795,1.0,1.0,1.0,1.0 -bm25,985,10,11.21996,1.0,1.0,1.0,1.0 -bm25,993,10,9.92417,0.5,1.0,0.33333,1.0 -bm25,1012,10,7.61842,1.0,1.0,1.0,1.0 -bm25,1014,10,9.47104,1.0,1.0,1.0,1.0 -bm25,1019,10,7.44254,0.63093,1.0,0.5,1.0 -bm25,1020,10,17.8105,0.5,1.0,0.33333,1.0 -bm25,1021,10,20.73158,0.5,1.0,0.33333,1.0 -bm25,1024,10,9.47762,1.0,1.0,1.0,1.0 -bm25,1029,10,17.35683,0.76536,0.66667,1.0,1.0 -bm25,1041,10,13.39667,0.41183,1.0,0.16667,0.0 -bm25,1049,10,9.07212,0.0,0.0,0.0,0.0 -bm25,1062,10,4.84004,1.0,1.0,1.0,1.0 -bm25,1086,10,15.71363,1.0,1.0,1.0,1.0 -bm25,1088,10,10.6375,0.0,0.0,0.0,0.0 -bm25,1089,10,13.75325,1.0,1.0,1.0,1.0 -bm25,1099,10,3.51079,0.0,0.0,0.0,0.0 -bm25,1100,10,3.27188,0.0,0.0,0.0,0.0 -bm25,1104,10,22.68571,1.0,1.0,1.0,1.0 -bm25,1107,10,8.28267,1.0,1.0,1.0,1.0 -bm25,1110,10,8.02971,0.0,0.0,0.0,0.0 -bm25,1121,10,10.59975,0.38685,1.0,0.2,0.0 -bm25,1130,10,13.90579,1.0,1.0,1.0,1.0 -bm25,1132,10,11.91846,0.0,0.0,0.0,0.0 -bm25,1137,10,8.67542,0.0,0.0,0.0,0.0 -bm25,1140,10,10.92612,0.30103,1.0,0.11111,0.0 -bm25,1144,10,18.85158,1.0,1.0,1.0,1.0 -bm25,1146,10,10.92546,1.0,1.0,1.0,1.0 -bm25,1150,10,11.66167,1.0,1.0,1.0,1.0 -bm25,1163,10,10.19796,1.0,1.0,1.0,1.0 -bm25,1175,10,7.95842,0.0,0.0,0.0,0.0 -bm25,1179,10,11.12475,0.0,0.0,0.0,0.0 -bm25,1180,10,9.48658,0.28906,1.0,0.1,0.0 -bm25,1185,10,26.67521,1.0,1.0,1.0,1.0 -bm25,1187,10,23.71529,1.0,1.0,1.0,1.0 -bm25,1191,10,10.41983,0.0,0.0,0.0,0.0 -bm25,1194,10,20.54817,0.0,0.0,0.0,0.0 -bm25,1196,10,12.18038,0.0,0.0,0.0,0.0 -bm25,1197,10,13.77562,0.0,0.0,0.0,0.0 -bm25,1199,10,16.18321,0.0,0.0,0.0,0.0 -bm25,1200,10,21.61933,0.0,0.0,0.0,0.0 -bm25,1202,10,14.239,0.0,0.0,0.0,0.0 -bm25,1204,10,13.75129,1.0,1.0,1.0,1.0 -bm25,1207,10,17.29383,0.35621,1.0,0.16667,0.0 -bm25,1213,10,12.81813,0.0,0.0,0.0,0.0 -bm25,1216,10,11.15983,1.0,1.0,1.0,1.0 -bm25,1221,10,15.0115,0.0,0.0,0.0,0.0 -bm25,1225,10,7.56033,1.0,1.0,1.0,1.0 -bm25,1226,10,15.0295,0.0,0.0,0.0,0.0 -bm25,1232,10,13.17204,1.0,1.0,1.0,1.0 -bm25,1241,10,8.33033,0.0,0.0,0.0,0.0 -bm25,1245,10,9.92354,0.0,0.0,0.0,0.0 -bm25,1259,10,18.17137,0.33333,1.0,0.14286,0.0 -bm25,1262,10,9.94996,0.63093,1.0,0.5,1.0 -bm25,1266,10,21.84096,0.63093,1.0,0.5,1.0 -bm25,1270,10,11.94404,1.0,1.0,1.0,1.0 -bm25,1271,10,16.55346,1.0,1.0,1.0,1.0 -bm25,1272,10,10.46617,0.0,0.0,0.0,0.0 -bm25,1273,10,10.57417,1.0,1.0,1.0,1.0 -bm25,1274,10,20.18742,0.71226,1.0,0.5,1.0 -bm25,1278,10,13.38575,0.0,0.0,0.0,0.0 -bm25,1279,10,11.68337,0.0,0.0,0.0,0.0 -bm25,1280,10,12.35929,0.0,0.0,0.0,0.0 -bm25,1281,10,9.50821,0.0,0.0,0.0,0.0 -bm25,1282,10,12.869,0.35621,1.0,0.16667,0.0 -bm25,1290,10,9.36837,1.0,1.0,1.0,1.0 -bm25,1292,10,9.74112,1.0,1.0,1.0,1.0 -bm25,1298,10,19.31408,1.0,1.0,1.0,1.0 -bm25,1303,10,6.09479,0.0,0.0,0.0,0.0 -bm25,1316,10,8.11121,0.0,0.0,0.0,0.0 -bm25,1319,10,8.89904,1.0,1.0,1.0,1.0 -bm25,1320,10,14.01721,1.0,1.0,1.0,1.0 -bm25,1332,10,13.21167,0.0,0.0,0.0,0.0 -bm25,1335,10,8.11525,1.0,1.0,1.0,1.0 -bm25,1336,10,6.30163,1.0,1.0,1.0,1.0 -bm25,1337,10,9.46729,1.0,1.0,1.0,1.0 -bm25,1339,10,13.09654,1.0,1.0,1.0,1.0 -bm25,1344,10,21.95104,0.63093,1.0,0.5,1.0 -bm25,1352,10,9.85783,1.0,1.0,1.0,1.0 -bm25,1359,10,16.778,1.0,1.0,1.0,1.0 -bm25,1362,10,6.74,0.0,0.0,0.0,0.0 -bm25,1363,10,9.49342,0.0,0.0,0.0,0.0 -bm25,1368,10,6.8545,0.0,0.0,0.0,0.0 -bm25,1370,10,8.11725,0.0,0.0,0.0,0.0 -bm25,1379,10,12.93425,0.41443,0.5,0.5,1.0 -bm25,1382,10,8.71808,0.0,0.0,0.0,0.0 -bm25,1385,10,4.69367,1.0,1.0,1.0,1.0 -bm25,1389,10,6.69988,1.0,1.0,1.0,1.0 -bm25,1395,10,19.72512,0.0,0.0,0.0,0.0 -dense+rerank,1,10,1123.79142,0.0,0.0,0.0,0.0 -dense+rerank,3,10,1032.48267,1.0,1.0,1.0,1.0 -dense+rerank,5,10,1109.01967,1.0,1.0,1.0,1.0 -dense+rerank,13,10,1103.68925,0.0,0.0,0.0,0.0 -dense+rerank,36,10,1077.97867,0.38896,1.0,0.14286,0.0 -dense+rerank,42,10,1074.05229,1.0,1.0,1.0,1.0 -dense+rerank,48,10,1106.43521,0.0,0.0,0.0,0.0 -dense+rerank,49,10,1076.86192,1.0,1.0,1.0,1.0 -dense+rerank,50,10,1152.32192,1.0,1.0,1.0,1.0 -dense+rerank,51,10,1114.718,1.0,1.0,1.0,1.0 -dense+rerank,53,10,1123.31592,1.0,1.0,1.0,1.0 -dense+rerank,54,10,1080.08425,0.35621,1.0,0.16667,0.0 -dense+rerank,56,10,1100.64671,1.0,1.0,1.0,1.0 -dense+rerank,57,10,1101.39783,1.0,1.0,1.0,1.0 -dense+rerank,70,10,1151.23329,0.0,0.0,0.0,0.0 -dense+rerank,72,10,1153.25692,1.0,1.0,1.0,1.0 -dense+rerank,75,10,1112.48379,1.0,1.0,1.0,1.0 -dense+rerank,94,10,1106.95029,0.31546,1.0,0.125,0.0 -dense+rerank,99,10,1137.70262,1.0,1.0,1.0,1.0 -dense+rerank,100,10,975.53646,1.0,1.0,1.0,1.0 -dense+rerank,113,10,1026.49671,1.0,1.0,1.0,1.0 -dense+rerank,115,10,1140.12192,1.0,1.0,1.0,1.0 -dense+rerank,118,10,1131.09175,1.0,1.0,1.0,1.0 -dense+rerank,124,10,1105.00454,1.0,1.0,1.0,1.0 -dense+rerank,127,10,1150.1555,0.63093,1.0,0.5,1.0 -dense+rerank,128,10,1114.48542,0.0,0.0,0.0,0.0 -dense+rerank,129,10,1101.4235,1.0,1.0,1.0,1.0 -dense+rerank,130,10,1165.80713,1.0,1.0,1.0,1.0 -dense+rerank,132,10,1175.55933,0.0,0.0,0.0,0.0 -dense+rerank,133,10,1151.89404,0.79725,0.8,1.0,1.0 -dense+rerank,137,10,1145.46288,1.0,1.0,1.0,1.0 -dense+rerank,141,10,1141.78171,0.91972,1.0,1.0,1.0 -dense+rerank,142,10,1142.21338,1.0,1.0,1.0,1.0 -dense+rerank,143,10,1146.65992,1.0,1.0,1.0,1.0 -dense+rerank,146,10,1125.278,1.0,1.0,1.0,1.0 -dense+rerank,148,10,1141.69921,1.0,1.0,1.0,1.0 -dense+rerank,163,10,1117.48492,0.63093,1.0,0.5,1.0 -dense+rerank,171,10,1157.83225,1.0,1.0,1.0,1.0 -dense+rerank,179,10,1168.137,0.90522,1.0,1.0,1.0 -dense+rerank,180,10,1151.58221,1.0,1.0,1.0,1.0 -dense+rerank,183,10,1079.27425,1.0,1.0,1.0,1.0 -dense+rerank,185,10,1134.59917,0.43068,1.0,0.25,0.0 -dense+rerank,198,10,1135.96313,0.0,0.0,0.0,0.0 -dense+rerank,208,10,1155.74363,1.0,1.0,1.0,1.0 -dense+rerank,212,10,1184.41038,0.5,1.0,0.33333,1.0 -dense+rerank,213,10,1152.37488,0.0,0.0,0.0,0.0 -dense+rerank,216,10,774.91633,1.0,1.0,1.0,1.0 -dense+rerank,217,10,1160.19504,1.0,1.0,1.0,1.0 -dense+rerank,218,10,1015.80796,1.0,1.0,1.0,1.0 -dense+rerank,219,10,1094.69896,1.0,1.0,1.0,1.0 -dense+rerank,230,10,1180.37588,1.0,1.0,1.0,1.0 -dense+rerank,232,10,1180.30567,0.63093,1.0,0.5,1.0 -dense+rerank,233,10,1129.36187,1.0,1.0,1.0,1.0 -dense+rerank,236,10,1187.07254,1.0,1.0,1.0,1.0 -dense+rerank,237,10,1073.7945,0.43068,1.0,0.25,0.0 -dense+rerank,238,10,1180.71437,0.0,0.0,0.0,0.0 -dense+rerank,239,10,1233.15046,0.0,0.0,0.0,0.0 -dense+rerank,248,10,1179.48346,1.0,1.0,1.0,1.0 -dense+rerank,249,10,1096.71629,1.0,1.0,1.0,1.0 -dense+rerank,261,10,1200.18354,0.38685,0.5,0.5,1.0 -dense+rerank,268,10,1163.59446,0.63093,1.0,0.5,1.0 -dense+rerank,269,10,1208.72838,0.63093,1.0,0.5,1.0 -dense+rerank,274,10,1161.69717,1.0,1.0,1.0,1.0 -dense+rerank,275,10,1217.37071,0.87108,1.0,1.0,1.0 -dense+rerank,279,10,1174.43017,1.0,1.0,1.0,1.0 -dense+rerank,294,10,763.11954,1.0,1.0,1.0,1.0 -dense+rerank,295,10,1186.82517,1.0,1.0,1.0,1.0 -dense+rerank,298,10,1188.27658,1.0,1.0,1.0,1.0 -dense+rerank,300,10,806.29587,0.5,1.0,0.33333,1.0 -dense+rerank,303,10,1223.99888,0.0,0.0,0.0,0.0 -dense+rerank,312,10,1057.40067,0.0,0.0,0.0,0.0 -dense+rerank,314,10,1203.28383,0.0,0.0,0.0,0.0 -dense+rerank,324,10,1168.70367,0.38685,1.0,0.2,0.0 -dense+rerank,327,10,1164.12988,1.0,1.0,1.0,1.0 -dense+rerank,338,10,1210.88517,1.0,1.0,1.0,1.0 -dense+rerank,343,10,1219.21533,0.57064,1.0,0.33333,1.0 -dense+rerank,350,10,1095.67363,1.0,1.0,1.0,1.0 -dense+rerank,354,10,1141.55521,1.0,1.0,1.0,1.0 -dense+rerank,362,10,1201.13904,1.0,1.0,1.0,1.0 -dense+rerank,380,10,1157.986,1.0,1.0,1.0,1.0 -dense+rerank,384,10,1209.70408,0.0,0.0,0.0,0.0 -dense+rerank,385,10,1129.46833,0.69343,1.0,0.5,1.0 -dense+rerank,386,10,1010.65254,1.0,1.0,1.0,1.0 -dense+rerank,388,10,1031.92321,0.5,1.0,0.33333,1.0 -dense+rerank,399,10,1226.45392,1.0,1.0,1.0,1.0 -dense+rerank,410,10,1218.72167,1.0,1.0,1.0,1.0 -dense+rerank,411,10,1151.06154,1.0,1.0,1.0,1.0 -dense+rerank,415,10,1194.30833,0.5,1.0,0.33333,1.0 -dense+rerank,421,10,1166.98192,0.0,0.0,0.0,0.0 -dense+rerank,431,10,1160.33221,0.30103,1.0,0.11111,0.0 -dense+rerank,436,10,644.32983,1.0,1.0,1.0,1.0 -dense+rerank,437,10,1213.83483,0.0,0.0,0.0,0.0 -dense+rerank,439,10,1153.60346,1.0,1.0,1.0,1.0 -dense+rerank,440,10,1207.20413,1.0,1.0,1.0,1.0 -dense+rerank,443,10,1199.29738,1.0,1.0,1.0,1.0 -dense+rerank,452,10,1272.42088,0.2372,0.5,0.2,0.0 -dense+rerank,475,10,1241.76658,0.38685,1.0,0.2,0.0 -dense+rerank,478,10,1166.55088,1.0,1.0,1.0,1.0 -dense+rerank,491,10,1204.89571,1.0,1.0,1.0,1.0 -dense+rerank,501,10,1231.85229,1.0,1.0,1.0,1.0 -dense+rerank,502,10,1106.589,0.0,0.0,0.0,0.0 -dense+rerank,507,10,1234.72412,0.63093,1.0,0.5,1.0 -dense+rerank,508,10,1177.19121,0.0,0.0,0.0,0.0 -dense+rerank,513,10,1273.34129,0.63093,1.0,0.5,1.0 -dense+rerank,514,10,1215.58454,0.5,1.0,0.33333,1.0 -dense+rerank,516,10,1221.75671,1.0,1.0,1.0,1.0 -dense+rerank,517,10,1287.08496,0.0,0.0,0.0,0.0 -dense+rerank,521,10,1222.85725,1.0,1.0,1.0,1.0 -dense+rerank,525,10,1225.68242,1.0,1.0,1.0,1.0 -dense+rerank,527,10,1276.91967,0.35621,1.0,0.16667,0.0 -dense+rerank,528,10,1200.24383,0.5,1.0,0.33333,1.0 -dense+rerank,532,10,1017.73696,1.0,1.0,1.0,1.0 -dense+rerank,533,10,1269.33033,1.0,1.0,1.0,1.0 -dense+rerank,535,10,1267.31738,0.0,0.0,0.0,0.0 -dense+rerank,536,10,1223.38237,1.0,1.0,1.0,1.0 -dense+rerank,539,10,1180.60712,1.0,1.0,1.0,1.0 -dense+rerank,540,10,1236.61354,0.30657,0.5,0.33333,1.0 -dense+rerank,544,10,1272.20812,0.38685,1.0,0.2,0.0 -dense+rerank,549,10,1162.25979,1.0,1.0,1.0,1.0 -dense+rerank,551,10,1134.31863,0.5,1.0,0.33333,1.0 -dense+rerank,552,10,1254.42571,1.0,1.0,1.0,1.0 -dense+rerank,554,10,1287.709,0.35621,1.0,0.16667,0.0 -dense+rerank,560,10,1016.27913,0.0,0.0,0.0,0.0 -dense+rerank,569,10,1253.39846,1.0,1.0,1.0,1.0 -dense+rerank,575,10,1094.32921,1.0,1.0,1.0,1.0 -dense+rerank,577,10,1237.64658,0.0,0.0,0.0,0.0 -dense+rerank,578,10,1283.61446,1.0,1.0,1.0,1.0 -dense+rerank,587,10,1228.42983,0.0,0.0,0.0,0.0 -dense+rerank,589,10,1264.34492,1.0,1.0,1.0,1.0 -dense+rerank,593,10,1313.3925,1.0,1.0,1.0,1.0 -dense+rerank,597,10,1317.65088,0.88546,1.0,1.0,1.0 -dense+rerank,598,10,1255.28896,0.33333,1.0,0.14286,0.0 -dense+rerank,613,10,1222.13438,1.0,1.0,1.0,1.0 -dense+rerank,619,10,1177.49212,0.61315,0.5,1.0,1.0 -dense+rerank,623,10,1231.70329,0.0,0.0,0.0,0.0 -dense+rerank,628,10,1221.88025,1.0,1.0,1.0,1.0 -dense+rerank,636,10,1256.3535,1.0,1.0,1.0,1.0 -dense+rerank,637,10,1306.50225,0.63093,1.0,0.5,1.0 -dense+rerank,641,10,1207.12729,0.79772,1.0,1.0,1.0 -dense+rerank,644,10,1024.95317,1.0,1.0,1.0,1.0 -dense+rerank,649,10,1278.44833,1.0,1.0,1.0,1.0 -dense+rerank,659,10,1224.42321,0.43068,1.0,0.25,0.0 -dense+rerank,660,10,1018.12954,0.0,0.0,0.0,0.0 -dense+rerank,674,10,1241.49804,1.0,1.0,1.0,1.0 -dense+rerank,684,10,1332.01888,0.43068,1.0,0.25,0.0 -dense+rerank,690,10,1267.83592,0.0,0.0,0.0,0.0 -dense+rerank,691,10,1263.50125,0.63093,1.0,0.5,1.0 -dense+rerank,692,10,1254.20613,1.0,1.0,1.0,1.0 -dense+rerank,693,10,1019.27167,1.0,1.0,1.0,1.0 -dense+rerank,700,10,1338.41037,0.33333,1.0,0.14286,0.0 -dense+rerank,702,10,1304.55204,0.31546,1.0,0.125,0.0 -dense+rerank,715,10,1210.27488,0.0,0.0,0.0,0.0 -dense+rerank,716,10,1289.25975,0.0,0.0,0.0,0.0 -dense+rerank,718,10,1318.40967,1.0,1.0,1.0,1.0 -dense+rerank,721,10,1164.36604,1.0,1.0,1.0,1.0 -dense+rerank,723,10,1206.86621,1.0,1.0,1.0,1.0 -dense+rerank,727,10,1354.23867,0.33333,1.0,0.14286,0.0 -dense+rerank,728,10,1295.01008,0.81753,1.0,1.0,1.0 -dense+rerank,729,10,1306.22275,1.0,1.0,1.0,1.0 -dense+rerank,742,10,1259.23683,1.0,1.0,1.0,1.0 -dense+rerank,743,10,1320.77721,1.0,1.0,1.0,1.0 -dense+rerank,744,10,1289.94204,0.63093,1.0,0.5,1.0 -dense+rerank,756,10,782.90746,0.43068,1.0,0.25,0.0 -dense+rerank,759,10,1147.06842,1.0,1.0,1.0,1.0 -dense+rerank,768,10,1356.92212,0.0,0.0,0.0,0.0 -dense+rerank,770,10,1346.4825,1.0,1.0,1.0,1.0 -dense+rerank,775,10,1091.58238,0.33333,1.0,0.14286,0.0 -dense+rerank,781,10,1204.31346,0.63093,1.0,0.5,1.0 -dense+rerank,783,10,1315.47383,0.31546,1.0,0.125,0.0 -dense+rerank,784,10,1367.59442,1.0,1.0,1.0,1.0 -dense+rerank,785,10,1271.25579,0.0,0.0,0.0,0.0 -dense+rerank,793,10,1374.2505,1.0,1.0,1.0,1.0 -dense+rerank,800,10,832.09312,0.63093,1.0,0.5,1.0 -dense+rerank,805,10,1367.24983,1.0,1.0,1.0,1.0 -dense+rerank,808,10,1035.65804,1.0,1.0,1.0,1.0 -dense+rerank,811,10,1331.49708,1.0,1.0,1.0,1.0 -dense+rerank,814,10,1298.90671,1.0,1.0,1.0,1.0 -dense+rerank,820,10,734.71379,0.0,0.0,0.0,0.0 -dense+rerank,821,10,715.60158,0.0,0.0,0.0,0.0 -dense+rerank,823,10,1386.79375,1.0,1.0,1.0,1.0 -dense+rerank,830,10,1280.23071,0.0,0.0,0.0,0.0 -dense+rerank,831,10,1211.38642,0.0,0.0,0.0,0.0 -dense+rerank,832,10,1099.81162,1.0,1.0,1.0,1.0 -dense+rerank,834,10,1390.78821,0.0,0.0,0.0,0.0 -dense+rerank,837,10,1286.14737,0.63093,1.0,0.5,1.0 -dense+rerank,839,10,1330.73671,1.0,1.0,1.0,1.0 -dense+rerank,845,10,1158.11633,1.0,1.0,1.0,1.0 -dense+rerank,847,10,1314.93787,1.0,1.0,1.0,1.0 -dense+rerank,852,10,1391.32725,0.63093,1.0,0.5,1.0 -dense+rerank,859,10,1309.40925,0.63093,1.0,0.5,1.0 -dense+rerank,870,10,1378.72292,0.0,0.0,0.0,0.0 -dense+rerank,873,10,1277.28154,0.41369,0.6,0.33333,1.0 -dense+rerank,879,10,1035.25763,1.0,1.0,1.0,1.0 -dense+rerank,880,10,1252.70358,1.0,1.0,1.0,1.0 -dense+rerank,882,10,1302.36271,1.0,1.0,1.0,1.0 -dense+rerank,887,10,1241.95175,0.0,0.0,0.0,0.0 -dense+rerank,903,10,1153.9285,1.0,1.0,1.0,1.0 -dense+rerank,904,10,1437.78129,1.0,1.0,1.0,1.0 -dense+rerank,907,10,1336.22767,1.0,1.0,1.0,1.0 -dense+rerank,911,10,1318.16671,1.0,1.0,1.0,1.0 -dense+rerank,913,10,1368.51012,0.0,0.0,0.0,0.0 -dense+rerank,914,10,1288.07604,0.0,0.0,0.0,0.0 -dense+rerank,921,10,1416.08108,1.0,1.0,1.0,1.0 -dense+rerank,922,10,1367.46408,1.0,1.0,1.0,1.0 -dense+rerank,936,10,1291.72254,1.0,1.0,1.0,1.0 -dense+rerank,956,10,1191.21333,1.0,1.0,1.0,1.0 -dense+rerank,957,10,782.3285,1.0,1.0,1.0,1.0 -dense+rerank,960,10,1042.81196,1.0,1.0,1.0,1.0 -dense+rerank,967,10,1378.92479,0.91972,1.0,1.0,1.0 -dense+rerank,971,10,1265.56425,0.95583,1.0,1.0,1.0 -dense+rerank,975,10,1356.11642,0.0,0.0,0.0,0.0 -dense+rerank,982,10,1223.87158,1.0,1.0,1.0,1.0 -dense+rerank,985,10,837.42871,1.0,1.0,1.0,1.0 -dense+rerank,993,10,1326.82354,1.0,1.0,1.0,1.0 -dense+rerank,1012,10,1300.78913,1.0,1.0,1.0,1.0 -dense+rerank,1014,10,1264.2005,1.0,1.0,1.0,1.0 -dense+rerank,1019,10,1335.16812,0.63093,1.0,0.5,1.0 -dense+rerank,1020,10,1392.76958,0.63093,1.0,0.5,1.0 -dense+rerank,1021,10,1523.63433,0.63093,1.0,0.5,1.0 -dense+rerank,1024,10,1171.01167,0.0,0.0,0.0,0.0 -dense+rerank,1029,10,1142.28096,0.76536,0.66667,1.0,1.0 -dense+rerank,1041,10,1480.74013,0.21841,0.5,0.16667,0.0 -dense+rerank,1049,10,1522.01663,0.43068,1.0,0.25,0.0 -dense+rerank,1062,10,1294.23071,1.0,1.0,1.0,1.0 -dense+rerank,1086,10,1186.11058,1.0,1.0,1.0,1.0 -dense+rerank,1088,10,1434.64033,1.0,1.0,1.0,1.0 -dense+rerank,1089,10,867.73158,1.0,1.0,1.0,1.0 -dense+rerank,1099,10,1204.57187,0.43068,1.0,0.25,0.0 -dense+rerank,1100,10,1400.83846,0.43068,1.0,0.25,0.0 -dense+rerank,1104,10,1383.66146,1.0,1.0,1.0,1.0 -dense+rerank,1107,10,1290.18037,1.0,1.0,1.0,1.0 -dense+rerank,1110,10,1014.35742,0.0,0.0,0.0,0.0 -dense+rerank,1121,10,1355.14646,1.0,1.0,1.0,1.0 -dense+rerank,1130,10,1263.18667,1.0,1.0,1.0,1.0 -dense+rerank,1132,10,1152.45962,0.61315,0.5,1.0,1.0 -dense+rerank,1137,10,1472.17875,1.0,1.0,1.0,1.0 -dense+rerank,1140,10,1464.82358,1.0,1.0,1.0,1.0 -dense+rerank,1144,10,1014.74312,1.0,1.0,1.0,1.0 -dense+rerank,1146,10,1429.43908,1.0,1.0,1.0,1.0 -dense+rerank,1150,10,1039.84946,1.0,1.0,1.0,1.0 -dense+rerank,1163,10,1421.78921,1.0,1.0,1.0,1.0 -dense+rerank,1175,10,1304.94808,0.0,0.0,0.0,0.0 -dense+rerank,1179,10,1346.11379,0.0,0.0,0.0,0.0 -dense+rerank,1180,10,1270.70462,1.0,1.0,1.0,1.0 -dense+rerank,1185,10,1378.10992,1.0,1.0,1.0,1.0 -dense+rerank,1187,10,850.40967,1.0,1.0,1.0,1.0 -dense+rerank,1191,10,802.57583,0.38685,1.0,0.2,0.0 -dense+rerank,1194,10,792.37929,1.0,1.0,1.0,1.0 -dense+rerank,1196,10,1441.02879,0.43068,1.0,0.25,0.0 -dense+rerank,1197,10,1382.16538,0.43068,1.0,0.25,0.0 -dense+rerank,1199,10,1355.71854,0.0,0.0,0.0,0.0 -dense+rerank,1200,10,1488.09796,0.0,0.0,0.0,0.0 -dense+rerank,1202,10,1432.58658,0.63093,1.0,0.5,1.0 -dense+rerank,1204,10,1341.20633,0.43068,1.0,0.25,0.0 -dense+rerank,1207,10,1410.52737,1.0,1.0,1.0,1.0 -dense+rerank,1213,10,1440.18763,0.0,0.0,0.0,0.0 -dense+rerank,1216,10,1339.29817,1.0,1.0,1.0,1.0 -dense+rerank,1221,10,1490.39733,0.38685,1.0,0.2,0.0 -dense+rerank,1225,10,1298.27137,1.0,1.0,1.0,1.0 -dense+rerank,1226,10,1503.97896,0.0,0.0,0.0,0.0 -dense+rerank,1232,10,1338.69542,1.0,1.0,1.0,1.0 -dense+rerank,1241,10,1265.85608,1.0,1.0,1.0,1.0 -dense+rerank,1245,10,1511.13012,0.0,0.0,0.0,0.0 -dense+rerank,1259,10,1402.32779,1.0,1.0,1.0,1.0 -dense+rerank,1262,10,1422.40629,1.0,1.0,1.0,1.0 -dense+rerank,1266,10,1276.94217,0.5,1.0,0.33333,1.0 -dense+rerank,1270,10,1364.24058,1.0,1.0,1.0,1.0 -dense+rerank,1271,10,1552.55583,1.0,1.0,1.0,1.0 -dense+rerank,1272,10,1565.85658,0.5,1.0,0.33333,1.0 -dense+rerank,1273,10,1398.25283,1.0,1.0,1.0,1.0 -dense+rerank,1274,10,1435.916,0.93252,1.0,1.0,1.0 -dense+rerank,1278,10,1311.95946,0.31546,1.0,0.125,0.0 -dense+rerank,1279,10,1261.17108,0.35621,1.0,0.16667,0.0 -dense+rerank,1280,10,1384.3155,0.35621,1.0,0.16667,0.0 -dense+rerank,1281,10,1509.63433,0.31546,1.0,0.125,0.0 -dense+rerank,1282,10,1317.51742,1.0,1.0,1.0,1.0 -dense+rerank,1290,10,1405.91438,1.0,1.0,1.0,1.0 -dense+rerank,1292,10,1510.95004,0.5,1.0,0.33333,1.0 -dense+rerank,1298,10,1497.57583,1.0,1.0,1.0,1.0 -dense+rerank,1303,10,1517.92283,1.0,1.0,1.0,1.0 -dense+rerank,1316,10,1493.45721,0.0,0.0,0.0,0.0 -dense+rerank,1319,10,1508.16358,1.0,1.0,1.0,1.0 -dense+rerank,1320,10,1404.60271,1.0,1.0,1.0,1.0 -dense+rerank,1332,10,1247.21688,0.0,0.0,0.0,0.0 -dense+rerank,1335,10,1438.0875,1.0,1.0,1.0,1.0 -dense+rerank,1336,10,1407.30042,1.0,1.0,1.0,1.0 -dense+rerank,1337,10,1448.51329,0.63093,1.0,0.5,1.0 -dense+rerank,1339,10,1440.82037,1.0,1.0,1.0,1.0 -dense+rerank,1344,10,1354.58608,0.5,1.0,0.33333,1.0 -dense+rerank,1352,10,1496.97158,1.0,1.0,1.0,1.0 -dense+rerank,1359,10,1601.553,1.0,1.0,1.0,1.0 -dense+rerank,1362,10,1339.81054,0.0,0.0,0.0,0.0 -dense+rerank,1363,10,1347.85971,0.0,0.0,0.0,0.0 -dense+rerank,1368,10,1549.46563,1.0,1.0,1.0,1.0 -dense+rerank,1370,10,1530.59171,0.63093,1.0,0.5,1.0 -dense+rerank,1379,10,1161.82746,0.83268,1.0,1.0,1.0 -dense+rerank,1382,10,1302.49587,1.0,1.0,1.0,1.0 -dense+rerank,1385,10,1345.51508,1.0,1.0,1.0,1.0 -dense+rerank,1389,10,1397.0295,1.0,1.0,1.0,1.0 -dense+rerank,1395,10,1520.75004,1.0,1.0,1.0,1.0 -hybrid+rerank,1,10,2090.27013,0.0,0.0,0.0,0.0 -hybrid+rerank,3,10,2010.44121,1.0,1.0,1.0,1.0 -hybrid+rerank,5,10,2408.31529,1.0,1.0,1.0,1.0 -hybrid+rerank,13,10,2227.87754,0.0,0.0,0.0,0.0 -hybrid+rerank,36,10,2136.61042,0.38896,1.0,0.14286,0.0 -hybrid+rerank,42,10,2332.12125,1.0,1.0,1.0,1.0 -hybrid+rerank,48,10,2510.38933,0.5,1.0,0.33333,1.0 -hybrid+rerank,49,10,1940.52475,1.0,1.0,1.0,1.0 -hybrid+rerank,50,10,1684.68137,1.0,1.0,1.0,1.0 -hybrid+rerank,51,10,2017.96217,0.63093,1.0,0.5,1.0 -hybrid+rerank,53,10,1849.28279,1.0,1.0,1.0,1.0 -hybrid+rerank,54,10,1667.66521,0.30103,1.0,0.11111,0.0 -hybrid+rerank,56,10,1979.10104,1.0,1.0,1.0,1.0 -hybrid+rerank,57,10,1893.61329,1.0,1.0,1.0,1.0 -hybrid+rerank,70,10,1712.0015,0.0,0.0,0.0,0.0 -hybrid+rerank,72,10,2110.36075,1.0,1.0,1.0,1.0 -hybrid+rerank,75,10,2019.49208,1.0,1.0,1.0,1.0 -hybrid+rerank,94,10,1648.26275,0.0,0.0,0.0,0.0 -hybrid+rerank,99,10,2204.94579,1.0,1.0,1.0,1.0 -hybrid+rerank,100,10,2056.77608,1.0,1.0,1.0,1.0 -hybrid+rerank,113,10,1612.507,1.0,1.0,1.0,1.0 -hybrid+rerank,115,10,1802.73954,1.0,1.0,1.0,1.0 -hybrid+rerank,118,10,1775.52592,1.0,1.0,1.0,1.0 -hybrid+rerank,124,10,1765.43708,1.0,1.0,1.0,1.0 -hybrid+rerank,127,10,2291.438,0.63093,1.0,0.5,1.0 -hybrid+rerank,128,10,1951.41054,0.0,0.0,0.0,0.0 -hybrid+rerank,129,10,2436.76763,1.0,1.0,1.0,1.0 -hybrid+rerank,130,10,1979.64983,1.0,1.0,1.0,1.0 -hybrid+rerank,132,10,2032.37179,0.0,0.0,0.0,0.0 -hybrid+rerank,133,10,2135.88304,0.51477,0.6,0.5,1.0 -hybrid+rerank,137,10,1608.78846,1.0,1.0,1.0,1.0 -hybrid+rerank,141,10,1722.16108,0.91972,1.0,1.0,1.0 -hybrid+rerank,142,10,1981.51921,1.0,1.0,1.0,1.0 -hybrid+rerank,143,10,1985.00258,1.0,1.0,1.0,1.0 -hybrid+rerank,146,10,1921.43592,1.0,1.0,1.0,1.0 -hybrid+rerank,148,10,1665.05342,1.0,1.0,1.0,1.0 -hybrid+rerank,163,10,1869.87633,0.63093,1.0,0.5,1.0 -hybrid+rerank,171,10,2008.43421,1.0,1.0,1.0,1.0 -hybrid+rerank,179,10,1801.68638,0.90522,1.0,1.0,1.0 -hybrid+rerank,180,10,2140.18867,1.0,1.0,1.0,1.0 -hybrid+rerank,183,10,1833.19154,1.0,1.0,1.0,1.0 -hybrid+rerank,185,10,1907.92658,0.43068,1.0,0.25,0.0 -hybrid+rerank,198,10,1670.9625,0.0,0.0,0.0,0.0 -hybrid+rerank,208,10,1975.89458,1.0,1.0,1.0,1.0 -hybrid+rerank,212,10,1777.58537,0.5,1.0,0.33333,1.0 -hybrid+rerank,213,10,2519.92317,0.0,0.0,0.0,0.0 -hybrid+rerank,216,10,2067.53579,1.0,1.0,1.0,1.0 -hybrid+rerank,217,10,1713.29779,1.0,1.0,1.0,1.0 -hybrid+rerank,218,10,2177.98033,1.0,1.0,1.0,1.0 -hybrid+rerank,219,10,1646.415,1.0,1.0,1.0,1.0 -hybrid+rerank,230,10,2037.36117,1.0,1.0,1.0,1.0 -hybrid+rerank,232,10,1549.17733,0.63093,1.0,0.5,1.0 -hybrid+rerank,233,10,1749.50217,1.0,1.0,1.0,1.0 -hybrid+rerank,236,10,1927.60525,1.0,1.0,1.0,1.0 -hybrid+rerank,237,10,2002.33171,0.43068,1.0,0.25,0.0 -hybrid+rerank,238,10,2044.15904,0.0,0.0,0.0,0.0 -hybrid+rerank,239,10,1906.41612,0.0,0.0,0.0,0.0 -hybrid+rerank,248,10,1735.49621,1.0,1.0,1.0,1.0 -hybrid+rerank,249,10,2072.76825,1.0,1.0,1.0,1.0 -hybrid+rerank,261,10,1579.78292,0.38685,0.5,0.5,1.0 -hybrid+rerank,268,10,1869.72333,0.63093,1.0,0.5,1.0 -hybrid+rerank,269,10,1846.1585,0.63093,1.0,0.5,1.0 -hybrid+rerank,274,10,2519.90437,1.0,1.0,1.0,1.0 -hybrid+rerank,275,10,1909.52717,0.87108,1.0,1.0,1.0 -hybrid+rerank,279,10,2075.65521,1.0,1.0,1.0,1.0 -hybrid+rerank,294,10,1984.4265,1.0,1.0,1.0,1.0 -hybrid+rerank,295,10,2007.31529,1.0,1.0,1.0,1.0 -hybrid+rerank,298,10,2061.11587,1.0,1.0,1.0,1.0 -hybrid+rerank,300,10,2045.63888,0.5,1.0,0.33333,1.0 -hybrid+rerank,303,10,1893.98512,0.0,0.0,0.0,0.0 -hybrid+rerank,312,10,1918.0075,0.0,0.0,0.0,0.0 -hybrid+rerank,314,10,2088.01942,0.0,0.0,0.0,0.0 -hybrid+rerank,324,10,2131.2455,0.30103,1.0,0.11111,0.0 -hybrid+rerank,327,10,2050.138,1.0,1.0,1.0,1.0 -hybrid+rerank,338,10,2726.83662,1.0,1.0,1.0,1.0 -hybrid+rerank,343,10,2563.9085,0.57064,1.0,0.33333,1.0 -hybrid+rerank,350,10,2038.675,1.0,1.0,1.0,1.0 -hybrid+rerank,354,10,2036.98754,1.0,1.0,1.0,1.0 -hybrid+rerank,362,10,1913.41625,1.0,1.0,1.0,1.0 -hybrid+rerank,380,10,1541.48213,1.0,1.0,1.0,1.0 -hybrid+rerank,384,10,1645.71412,0.0,0.0,0.0,0.0 -hybrid+rerank,385,10,1914.24683,0.65092,1.0,0.5,1.0 -hybrid+rerank,386,10,1955.12871,1.0,1.0,1.0,1.0 -hybrid+rerank,388,10,1983.46933,0.30103,1.0,0.11111,0.0 -hybrid+rerank,399,10,1719.79825,1.0,1.0,1.0,1.0 -hybrid+rerank,410,10,1403.68129,1.0,1.0,1.0,1.0 -hybrid+rerank,411,10,1646.98337,1.0,1.0,1.0,1.0 -hybrid+rerank,415,10,1625.32275,0.5,1.0,0.33333,1.0 -hybrid+rerank,421,10,1820.59046,0.0,0.0,0.0,0.0 -hybrid+rerank,431,10,1641.22817,0.0,0.0,0.0,0.0 -hybrid+rerank,436,10,1735.14175,1.0,1.0,1.0,1.0 -hybrid+rerank,437,10,1768.60996,0.0,0.0,0.0,0.0 -hybrid+rerank,439,10,1737.80158,1.0,1.0,1.0,1.0 -hybrid+rerank,440,10,1615.71771,1.0,1.0,1.0,1.0 -hybrid+rerank,443,10,1660.83675,1.0,1.0,1.0,1.0 -hybrid+rerank,452,10,1734.74379,0.60526,1.0,0.5,1.0 -hybrid+rerank,475,10,1651.56329,0.38685,1.0,0.2,0.0 -hybrid+rerank,478,10,1701.759,1.0,1.0,1.0,1.0 -hybrid+rerank,491,10,1858.05954,1.0,1.0,1.0,1.0 -hybrid+rerank,501,10,1505.67358,1.0,1.0,1.0,1.0 -hybrid+rerank,502,10,1905.43829,0.0,0.0,0.0,0.0 -hybrid+rerank,507,10,1616.59108,0.63093,1.0,0.5,1.0 -hybrid+rerank,508,10,1573.53746,0.0,0.0,0.0,0.0 -hybrid+rerank,513,10,1864.44138,0.63093,1.0,0.5,1.0 -hybrid+rerank,514,10,1723.42604,0.5,1.0,0.33333,1.0 -hybrid+rerank,516,10,1363.56175,1.0,1.0,1.0,1.0 -hybrid+rerank,517,10,1946.12846,0.0,0.0,0.0,0.0 -hybrid+rerank,521,10,2027.91887,1.0,1.0,1.0,1.0 -hybrid+rerank,525,10,1613.39079,1.0,1.0,1.0,1.0 -hybrid+rerank,527,10,1777.60146,0.0,0.0,0.0,0.0 -hybrid+rerank,528,10,1888.90408,0.5,1.0,0.33333,1.0 -hybrid+rerank,532,10,2171.88321,1.0,1.0,1.0,1.0 -hybrid+rerank,533,10,1874.37408,1.0,1.0,1.0,1.0 -hybrid+rerank,535,10,1694.07637,0.0,0.0,0.0,0.0 -hybrid+rerank,536,10,1957.14954,1.0,1.0,1.0,1.0 -hybrid+rerank,539,10,1667.14725,1.0,1.0,1.0,1.0 -hybrid+rerank,540,10,1521.31008,0.30657,0.5,0.33333,1.0 -hybrid+rerank,544,10,1730.57583,0.38685,1.0,0.2,0.0 -hybrid+rerank,549,10,1675.22567,1.0,1.0,1.0,1.0 -hybrid+rerank,551,10,1485.90633,0.5,1.0,0.33333,1.0 -hybrid+rerank,552,10,1620.34275,1.0,1.0,1.0,1.0 -hybrid+rerank,554,10,1646.75838,0.30103,1.0,0.11111,0.0 -hybrid+rerank,560,10,1608.01358,0.0,0.0,0.0,0.0 -hybrid+rerank,569,10,1415.37083,1.0,1.0,1.0,1.0 -hybrid+rerank,575,10,1871.85413,1.0,1.0,1.0,1.0 -hybrid+rerank,577,10,1856.17796,0.0,0.0,0.0,0.0 -hybrid+rerank,578,10,1792.79525,1.0,1.0,1.0,1.0 -hybrid+rerank,587,10,1748.41221,0.63093,1.0,0.5,1.0 -hybrid+rerank,589,10,1926.57525,1.0,1.0,1.0,1.0 -hybrid+rerank,593,10,1954.00621,1.0,1.0,1.0,1.0 -hybrid+rerank,597,10,1640.59283,0.88546,1.0,1.0,1.0 -hybrid+rerank,598,10,1496.493,0.33333,1.0,0.14286,0.0 -hybrid+rerank,613,10,1748.33842,1.0,1.0,1.0,1.0 -hybrid+rerank,619,10,1879.96133,0.38685,0.5,0.5,1.0 -hybrid+rerank,623,10,1880.93779,0.0,0.0,0.0,0.0 -hybrid+rerank,628,10,1906.13812,1.0,1.0,1.0,1.0 -hybrid+rerank,636,10,1807.186,1.0,1.0,1.0,1.0 -hybrid+rerank,637,10,1828.02658,0.63093,1.0,0.5,1.0 -hybrid+rerank,641,10,1845.38108,0.79772,1.0,1.0,1.0 -hybrid+rerank,644,10,1702.25558,1.0,1.0,1.0,1.0 -hybrid+rerank,649,10,1991.12733,1.0,1.0,1.0,1.0 -hybrid+rerank,659,10,1681.25429,0.35621,1.0,0.16667,0.0 -hybrid+rerank,660,10,2132.37383,0.0,0.0,0.0,0.0 -hybrid+rerank,674,10,1766.96696,1.0,1.0,1.0,1.0 -hybrid+rerank,684,10,1673.49567,0.43068,1.0,0.25,0.0 -hybrid+rerank,690,10,2042.92654,1.0,1.0,1.0,1.0 -hybrid+rerank,691,10,1923.85317,0.63093,1.0,0.5,1.0 -hybrid+rerank,692,10,1855.48096,1.0,1.0,1.0,1.0 -hybrid+rerank,693,10,2001.52625,1.0,1.0,1.0,1.0 -hybrid+rerank,700,10,1879.43017,0.33333,1.0,0.14286,0.0 -hybrid+rerank,702,10,1830.93671,0.31546,1.0,0.125,0.0 -hybrid+rerank,715,10,1984.67242,0.33333,1.0,0.14286,0.0 -hybrid+rerank,716,10,1843.64688,0.0,0.0,0.0,0.0 -hybrid+rerank,718,10,2130.37004,0.63093,1.0,0.5,1.0 -hybrid+rerank,721,10,1930.56496,1.0,1.0,1.0,1.0 -hybrid+rerank,723,10,2040.30267,1.0,1.0,1.0,1.0 -hybrid+rerank,727,10,1831.10775,0.31546,1.0,0.125,0.0 -hybrid+rerank,728,10,1771.497,0.80657,1.0,1.0,1.0 -hybrid+rerank,729,10,2075.54033,1.0,1.0,1.0,1.0 -hybrid+rerank,742,10,2176.91442,1.0,1.0,1.0,1.0 -hybrid+rerank,743,10,2233.65008,1.0,1.0,1.0,1.0 -hybrid+rerank,744,10,1977.66183,0.63093,1.0,0.5,1.0 -hybrid+rerank,756,10,1701.31379,0.43068,1.0,0.25,0.0 -hybrid+rerank,759,10,1632.10454,1.0,1.0,1.0,1.0 -hybrid+rerank,768,10,2073.28892,0.0,0.0,0.0,0.0 -hybrid+rerank,770,10,2359.30912,1.0,1.0,1.0,1.0 -hybrid+rerank,775,10,1748.27492,0.33333,1.0,0.14286,0.0 -hybrid+rerank,781,10,2052.22308,0.63093,1.0,0.5,1.0 -hybrid+rerank,783,10,2025.13821,0.28906,1.0,0.1,0.0 -hybrid+rerank,784,10,2053.49696,1.0,1.0,1.0,1.0 -hybrid+rerank,785,10,1901.22742,0.28906,1.0,0.1,0.0 -hybrid+rerank,793,10,1947.63167,1.0,1.0,1.0,1.0 -hybrid+rerank,800,10,1952.64079,0.63093,1.0,0.5,1.0 -hybrid+rerank,805,10,2006.04583,1.0,1.0,1.0,1.0 -hybrid+rerank,808,10,1955.03763,1.0,1.0,1.0,1.0 -hybrid+rerank,811,10,1857.52904,1.0,1.0,1.0,1.0 -hybrid+rerank,814,10,2230.52508,1.0,1.0,1.0,1.0 -hybrid+rerank,820,10,2076.40117,0.35621,1.0,0.16667,0.0 -hybrid+rerank,821,10,1927.13742,0.38685,1.0,0.2,0.0 -hybrid+rerank,823,10,2076.31371,1.0,1.0,1.0,1.0 -hybrid+rerank,830,10,2069.50717,0.0,0.0,0.0,0.0 -hybrid+rerank,831,10,2257.04883,0.0,0.0,0.0,0.0 -hybrid+rerank,832,10,2058.49158,1.0,1.0,1.0,1.0 -hybrid+rerank,834,10,2135.10392,0.38685,1.0,0.2,0.0 -hybrid+rerank,837,10,1985.80842,0.63093,1.0,0.5,1.0 -hybrid+rerank,839,10,1830.47492,1.0,1.0,1.0,1.0 -hybrid+rerank,845,10,1668.53771,1.0,1.0,1.0,1.0 -hybrid+rerank,847,10,1713.83288,1.0,1.0,1.0,1.0 -hybrid+rerank,852,10,2161.18183,0.63093,1.0,0.5,1.0 -hybrid+rerank,859,10,2169.11758,0.63093,1.0,0.5,1.0 -hybrid+rerank,870,10,1790.34392,0.0,0.0,0.0,0.0 -hybrid+rerank,873,10,2033.87179,0.25202,0.4,0.2,0.0 -hybrid+rerank,879,10,2323.33313,1.0,1.0,1.0,1.0 -hybrid+rerank,880,10,2388.20787,1.0,1.0,1.0,1.0 -hybrid+rerank,882,10,1964.70054,1.0,1.0,1.0,1.0 -hybrid+rerank,887,10,2043.975,0.0,0.0,0.0,0.0 -hybrid+rerank,903,10,1975.62554,1.0,1.0,1.0,1.0 -hybrid+rerank,904,10,1791.522,1.0,1.0,1.0,1.0 -hybrid+rerank,907,10,1987.95929,1.0,1.0,1.0,1.0 -hybrid+rerank,911,10,2137.05721,1.0,1.0,1.0,1.0 -hybrid+rerank,913,10,2253.78683,0.0,0.0,0.0,0.0 -hybrid+rerank,914,10,2184.30963,0.0,0.0,0.0,0.0 -hybrid+rerank,921,10,2007.84025,1.0,1.0,1.0,1.0 -hybrid+rerank,922,10,1925.63483,1.0,1.0,1.0,1.0 -hybrid+rerank,936,10,2277.14917,1.0,1.0,1.0,1.0 -hybrid+rerank,956,10,2257.80517,1.0,1.0,1.0,1.0 -hybrid+rerank,957,10,1293.80929,1.0,1.0,1.0,1.0 -hybrid+rerank,960,10,2246.12925,1.0,1.0,1.0,1.0 -hybrid+rerank,967,10,1707.28317,0.91972,1.0,1.0,1.0 -hybrid+rerank,971,10,1523.04808,0.95583,1.0,1.0,1.0 -hybrid+rerank,975,10,1937.28458,0.0,0.0,0.0,0.0 -hybrid+rerank,982,10,1867.97454,1.0,1.0,1.0,1.0 -hybrid+rerank,985,10,2098.27108,1.0,1.0,1.0,1.0 -hybrid+rerank,993,10,1893.69758,1.0,1.0,1.0,1.0 -hybrid+rerank,1012,10,1922.34408,1.0,1.0,1.0,1.0 -hybrid+rerank,1014,10,2047.818,1.0,1.0,1.0,1.0 -hybrid+rerank,1019,10,1732.17921,0.63093,1.0,0.5,1.0 -hybrid+rerank,1020,10,1737.86654,0.63093,1.0,0.5,1.0 -hybrid+rerank,1021,10,1675.41483,0.63093,1.0,0.5,1.0 -hybrid+rerank,1024,10,1871.39908,1.0,1.0,1.0,1.0 -hybrid+rerank,1029,10,1590.53333,0.76536,0.66667,1.0,1.0 -hybrid+rerank,1041,10,1343.32067,0.21841,0.5,0.16667,0.0 -hybrid+rerank,1049,10,2000.37804,0.43068,1.0,0.25,0.0 -hybrid+rerank,1062,10,1521.338,1.0,1.0,1.0,1.0 -hybrid+rerank,1086,10,2364.48154,1.0,1.0,1.0,1.0 -hybrid+rerank,1088,10,1731.91571,1.0,1.0,1.0,1.0 -hybrid+rerank,1089,10,1533.21892,1.0,1.0,1.0,1.0 -hybrid+rerank,1099,10,1678.943,0.43068,1.0,0.25,0.0 -hybrid+rerank,1100,10,1625.54188,0.43068,1.0,0.25,0.0 -hybrid+rerank,1104,10,1976.7515,1.0,1.0,1.0,1.0 -hybrid+rerank,1107,10,1629.14042,1.0,1.0,1.0,1.0 -hybrid+rerank,1110,10,1981.11683,0.0,0.0,0.0,0.0 -hybrid+rerank,1121,10,1543.38996,1.0,1.0,1.0,1.0 -hybrid+rerank,1130,10,1735.64625,1.0,1.0,1.0,1.0 -hybrid+rerank,1132,10,1691.36554,0.61315,0.5,1.0,1.0 -hybrid+rerank,1137,10,1890.60912,1.0,1.0,1.0,1.0 -hybrid+rerank,1140,10,1567.69883,1.0,1.0,1.0,1.0 -hybrid+rerank,1144,10,1982.03229,1.0,1.0,1.0,1.0 -hybrid+rerank,1146,10,1657.43742,1.0,1.0,1.0,1.0 -hybrid+rerank,1150,10,1696.84937,1.0,1.0,1.0,1.0 -hybrid+rerank,1163,10,1787.73462,1.0,1.0,1.0,1.0 -hybrid+rerank,1175,10,1826.04033,0.0,0.0,0.0,0.0 -hybrid+rerank,1179,10,1815.64225,0.0,0.0,0.0,0.0 -hybrid+rerank,1180,10,1692.61196,1.0,1.0,1.0,1.0 -hybrid+rerank,1185,10,2141.43963,1.0,1.0,1.0,1.0 -hybrid+rerank,1187,10,1793.47896,1.0,1.0,1.0,1.0 -hybrid+rerank,1191,10,1618.78787,0.0,0.0,0.0,0.0 -hybrid+rerank,1194,10,1753.14062,1.0,1.0,1.0,1.0 -hybrid+rerank,1196,10,1691.73533,0.43068,1.0,0.25,0.0 -hybrid+rerank,1197,10,1783.07375,0.43068,1.0,0.25,0.0 -hybrid+rerank,1199,10,1984.14492,0.0,0.0,0.0,0.0 -hybrid+rerank,1200,10,1935.74621,0.0,0.0,0.0,0.0 -hybrid+rerank,1202,10,1788.83104,0.63093,1.0,0.5,1.0 -hybrid+rerank,1204,10,1739.28363,0.43068,1.0,0.25,0.0 -hybrid+rerank,1207,10,1665.95154,1.0,1.0,1.0,1.0 -hybrid+rerank,1213,10,1648.99808,0.0,0.0,0.0,0.0 -hybrid+rerank,1216,10,1750.03933,1.0,1.0,1.0,1.0 -hybrid+rerank,1221,10,1853.61367,0.31546,1.0,0.125,0.0 -hybrid+rerank,1225,10,1669.35279,1.0,1.0,1.0,1.0 -hybrid+rerank,1226,10,1910.41467,0.0,0.0,0.0,0.0 -hybrid+rerank,1232,10,1940.87062,1.0,1.0,1.0,1.0 -hybrid+rerank,1241,10,1746.01446,1.0,1.0,1.0,1.0 -hybrid+rerank,1245,10,2009.14504,0.0,0.0,0.0,0.0 -hybrid+rerank,1259,10,2176.68571,1.0,1.0,1.0,1.0 -hybrid+rerank,1262,10,1588.03508,1.0,1.0,1.0,1.0 -hybrid+rerank,1266,10,1491.67237,0.5,1.0,0.33333,1.0 -hybrid+rerank,1270,10,1860.53738,1.0,1.0,1.0,1.0 -hybrid+rerank,1271,10,1821.46392,1.0,1.0,1.0,1.0 -hybrid+rerank,1272,10,1800.95825,0.5,1.0,0.33333,1.0 -hybrid+rerank,1273,10,1607.84646,1.0,1.0,1.0,1.0 -hybrid+rerank,1274,10,1686.29513,0.93252,1.0,1.0,1.0 -hybrid+rerank,1278,10,2042.30229,0.0,0.0,0.0,0.0 -hybrid+rerank,1279,10,2002.05467,0.28906,1.0,0.1,0.0 -hybrid+rerank,1280,10,1801.96192,0.0,0.0,0.0,0.0 -hybrid+rerank,1281,10,1896.63333,0.0,0.0,0.0,0.0 -hybrid+rerank,1282,10,2106.57788,1.0,1.0,1.0,1.0 -hybrid+rerank,1290,10,2012.7775,1.0,1.0,1.0,1.0 -hybrid+rerank,1292,10,1867.94567,0.5,1.0,0.33333,1.0 -hybrid+rerank,1298,10,2114.35192,1.0,1.0,1.0,1.0 -hybrid+rerank,1303,10,1939.34979,1.0,1.0,1.0,1.0 -hybrid+rerank,1316,10,1742.88963,0.0,0.0,0.0,0.0 -hybrid+rerank,1319,10,1708.16721,1.0,1.0,1.0,1.0 -hybrid+rerank,1320,10,1502.92692,1.0,1.0,1.0,1.0 -hybrid+rerank,1332,10,1768.65233,0.0,0.0,0.0,0.0 -hybrid+rerank,1335,10,1630.21571,1.0,1.0,1.0,1.0 -hybrid+rerank,1336,10,1427.86004,1.0,1.0,1.0,1.0 -hybrid+rerank,1337,10,1626.50367,0.63093,1.0,0.5,1.0 -hybrid+rerank,1339,10,1902.04538,1.0,1.0,1.0,1.0 -hybrid+rerank,1344,10,1694.55921,0.5,1.0,0.33333,1.0 -hybrid+rerank,1352,10,1988.31883,1.0,1.0,1.0,1.0 -hybrid+rerank,1359,10,2264.59563,1.0,1.0,1.0,1.0 -hybrid+rerank,1362,10,1936.28804,0.0,0.0,0.0,0.0 -hybrid+rerank,1363,10,1626.05375,0.0,0.0,0.0,0.0 -hybrid+rerank,1368,10,1548.69996,1.0,1.0,1.0,1.0 -hybrid+rerank,1370,10,1541.69537,0.63093,1.0,0.5,1.0 -hybrid+rerank,1379,10,1691.18821,0.83268,1.0,1.0,1.0 -hybrid+rerank,1382,10,1729.26863,1.0,1.0,1.0,1.0 -hybrid+rerank,1385,10,1760.07896,1.0,1.0,1.0,1.0 -hybrid+rerank,1389,10,1853.72542,1.0,1.0,1.0,1.0 -hybrid+rerank,1395,10,2103.66529,1.0,1.0,1.0,1.0 -app_default,1,0,710.89904,0.0,0.0,0.0,0.0 -app_default,3,3,413.00508,1.0,1.0,1.0,1.0 -app_default,5,1,769.69679,1.0,1.0,1.0,1.0 -app_default,13,2,579.52083,0.0,0.0,0.0,0.0 -app_default,36,3,654.99896,0.0,0.0,0.0,0.0 -app_default,42,1,605.51696,1.0,1.0,1.0,1.0 -app_default,48,2,722.91617,0.0,0.0,0.0,0.0 -app_default,49,2,588.27112,1.0,1.0,1.0,1.0 -app_default,50,3,396.56158,1.0,1.0,1.0,1.0 -app_default,51,3,722.46796,0.63093,1.0,0.5,1.0 -app_default,53,3,648.88954,1.0,1.0,1.0,1.0 -app_default,54,3,645.14383,0.0,0.0,0.0,0.0 -app_default,56,2,648.84117,1.0,1.0,1.0,1.0 -app_default,57,2,749.72825,1.0,1.0,1.0,1.0 -app_default,70,3,620.27692,0.0,0.0,0.0,0.0 -app_default,72,0,739.98546,0.0,0.0,0.0,0.0 -app_default,75,1,494.71808,1.0,1.0,1.0,1.0 -app_default,94,0,728.17463,0.0,0.0,0.0,0.0 -app_default,99,1,762.34496,1.0,1.0,1.0,1.0 -app_default,100,3,382.87029,1.0,1.0,1.0,1.0 -app_default,113,3,609.51804,1.0,1.0,1.0,1.0 -app_default,115,1,506.59,1.0,1.0,1.0,1.0 -app_default,118,3,529.55096,1.0,1.0,1.0,1.0 -app_default,124,3,632.37592,1.0,1.0,1.0,1.0 -app_default,127,2,690.65087,0.63093,1.0,0.5,1.0 -app_default,128,0,787.20508,0.0,0.0,0.0,0.0 -app_default,129,1,738.30858,1.0,1.0,1.0,1.0 -app_default,130,1,717.58429,1.0,1.0,1.0,1.0 -app_default,132,3,609.91037,0.0,0.0,0.0,0.0 -app_default,133,2,613.28221,0.55315,0.4,1.0,1.0 -app_default,137,1,769.01612,1.0,1.0,1.0,1.0 -app_default,141,3,570.54796,0.91972,1.0,1.0,1.0 -app_default,142,3,779.67083,1.0,1.0,1.0,1.0 -app_default,143,3,774.94588,1.0,1.0,1.0,1.0 -app_default,146,3,625.98388,1.0,1.0,1.0,1.0 -app_default,148,3,446.30367,1.0,1.0,1.0,1.0 -app_default,163,3,502.38696,0.63093,1.0,0.5,1.0 -app_default,171,3,553.88104,1.0,1.0,1.0,1.0 -app_default,179,3,543.27704,0.63668,0.5,1.0,1.0 -app_default,180,2,627.79458,1.0,1.0,1.0,1.0 -app_default,183,3,483.10713,1.0,1.0,1.0,1.0 -app_default,185,3,714.22571,0.5,1.0,0.33333,1.0 -app_default,198,0,807.00779,0.0,0.0,0.0,0.0 -app_default,208,3,711.49729,1.0,1.0,1.0,1.0 -app_default,212,3,692.35958,0.5,1.0,0.33333,1.0 -app_default,213,3,563.46,0.0,0.0,0.0,0.0 -app_default,216,3,715.60692,1.0,1.0,1.0,1.0 -app_default,217,3,659.84033,1.0,1.0,1.0,1.0 -app_default,218,3,587.67067,1.0,1.0,1.0,1.0 -app_default,219,3,593.44913,1.0,1.0,1.0,1.0 -app_default,230,0,665.00142,0.0,0.0,0.0,0.0 -app_default,232,3,555.54446,0.63093,1.0,0.5,1.0 -app_default,233,1,478.67146,1.0,1.0,1.0,1.0 -app_default,236,1,502.23617,1.0,1.0,1.0,1.0 -app_default,237,3,383.64687,0.0,0.0,0.0,0.0 -app_default,238,1,532.00979,0.0,0.0,0.0,0.0 -app_default,239,1,430.27671,0.0,0.0,0.0,0.0 -app_default,248,1,738.29029,1.0,1.0,1.0,1.0 -app_default,249,1,757.63317,1.0,1.0,1.0,1.0 -app_default,261,3,620.93088,0.38685,0.5,0.5,1.0 -app_default,268,2,625.38437,0.63093,1.0,0.5,1.0 -app_default,269,1,435.22237,0.0,0.0,0.0,0.0 -app_default,274,3,604.8625,1.0,1.0,1.0,1.0 -app_default,275,3,636.04067,0.70392,0.66667,1.0,1.0 -app_default,279,3,658.26746,1.0,1.0,1.0,1.0 -app_default,294,1,469.923,0.0,0.0,0.0,0.0 -app_default,295,3,359.86408,1.0,1.0,1.0,1.0 -app_default,298,3,749.65663,1.0,1.0,1.0,1.0 -app_default,300,3,716.17946,0.5,1.0,0.33333,1.0 -app_default,303,1,692.35596,0.0,0.0,0.0,0.0 -app_default,312,2,667.37988,0.0,0.0,0.0,0.0 -app_default,314,2,711.36392,0.0,0.0,0.0,0.0 -app_default,324,1,675.02358,0.0,0.0,0.0,0.0 -app_default,327,2,741.16167,1.0,1.0,1.0,1.0 -app_default,338,3,647.45313,1.0,1.0,1.0,1.0 -app_default,343,3,638.08913,0.30657,0.5,0.33333,1.0 -app_default,350,1,667.06529,1.0,1.0,1.0,1.0 -app_default,354,2,401.26521,1.0,1.0,1.0,1.0 -app_default,362,3,695.29404,1.0,1.0,1.0,1.0 -app_default,380,2,685.91967,1.0,1.0,1.0,1.0 -app_default,384,2,643.59925,0.0,0.0,0.0,0.0 -app_default,385,3,717.74629,0.38685,0.5,0.5,1.0 -app_default,386,1,705.91433,1.0,1.0,1.0,1.0 -app_default,388,0,723.90933,0.0,0.0,0.0,0.0 -app_default,399,3,588.99183,1.0,1.0,1.0,1.0 -app_default,410,3,613.20108,1.0,1.0,1.0,1.0 -app_default,411,3,485.14546,1.0,1.0,1.0,1.0 -app_default,415,3,602.91071,0.63093,1.0,0.5,1.0 -app_default,421,0,823.35983,0.0,0.0,0.0,0.0 -app_default,431,3,513.93446,0.0,0.0,0.0,0.0 -app_default,436,1,632.08083,1.0,1.0,1.0,1.0 -app_default,437,3,702.146,0.0,0.0,0.0,0.0 -app_default,439,2,713.74021,1.0,1.0,1.0,1.0 -app_default,440,1,766.0835,1.0,1.0,1.0,1.0 -app_default,443,3,591.113,1.0,1.0,1.0,1.0 -app_default,452,3,734.54308,0.38685,0.5,0.5,1.0 -app_default,475,0,642.28475,0.0,0.0,0.0,0.0 -app_default,478,1,862.87312,1.0,1.0,1.0,1.0 -app_default,491,1,739.39129,1.0,1.0,1.0,1.0 -app_default,501,2,763.54979,1.0,1.0,1.0,1.0 -app_default,502,0,752.16975,0.0,0.0,0.0,0.0 -app_default,507,3,638.62071,0.63093,1.0,0.5,1.0 -app_default,508,1,702.62608,0.0,0.0,0.0,0.0 -app_default,513,2,699.924,0.63093,1.0,0.5,1.0 -app_default,514,3,704.74846,0.5,1.0,0.33333,1.0 -app_default,516,3,586.20071,1.0,1.0,1.0,1.0 -app_default,517,3,795.43483,0.0,0.0,0.0,0.0 -app_default,521,1,549.33933,1.0,1.0,1.0,1.0 -app_default,525,3,548.22379,1.0,1.0,1.0,1.0 -app_default,527,2,717.67779,0.0,0.0,0.0,0.0 -app_default,528,3,583.8285,0.5,1.0,0.33333,1.0 -app_default,532,1,705.06571,1.0,1.0,1.0,1.0 -app_default,533,1,719.31488,1.0,1.0,1.0,1.0 -app_default,535,3,748.59825,0.0,0.0,0.0,0.0 -app_default,536,1,776.391,1.0,1.0,1.0,1.0 -app_default,539,3,511.91563,1.0,1.0,1.0,1.0 -app_default,540,3,602.20479,0.30657,0.5,0.33333,1.0 -app_default,544,2,747.77108,0.0,0.0,0.0,0.0 -app_default,549,3,744.98496,1.0,1.0,1.0,1.0 -app_default,551,1,734.66463,0.0,0.0,0.0,0.0 -app_default,552,3,394.87225,1.0,1.0,1.0,1.0 -app_default,554,3,420.15333,0.0,0.0,0.0,0.0 -app_default,560,3,694.40908,0.0,0.0,0.0,0.0 -app_default,569,3,727.59788,1.0,1.0,1.0,1.0 -app_default,575,1,788.94696,1.0,1.0,1.0,1.0 -app_default,577,0,666.72121,0.0,0.0,0.0,0.0 -app_default,578,3,779.14954,1.0,1.0,1.0,1.0 -app_default,587,1,784.01404,0.0,0.0,0.0,0.0 -app_default,589,1,730.17233,1.0,1.0,1.0,1.0 -app_default,593,2,626.78912,1.0,1.0,1.0,1.0 -app_default,597,3,473.78346,0.70392,0.66667,1.0,1.0 -app_default,598,3,424.41738,0.0,0.0,0.0,0.0 -app_default,613,1,664.9115,1.0,1.0,1.0,1.0 -app_default,619,0,803.71562,0.0,0.0,0.0,0.0 -app_default,623,3,583.37146,0.0,0.0,0.0,0.0 -app_default,628,3,727.52525,1.0,1.0,1.0,1.0 -app_default,636,3,685.03596,1.0,1.0,1.0,1.0 -app_default,637,1,774.64783,0.0,0.0,0.0,0.0 -app_default,641,1,649.32296,0.61315,0.5,1.0,1.0 -app_default,644,2,723.36954,1.0,1.0,1.0,1.0 -app_default,649,0,765.96204,0.0,0.0,0.0,0.0 -app_default,659,0,774.41746,0.0,0.0,0.0,0.0 -app_default,660,0,807.41288,0.0,0.0,0.0,0.0 -app_default,674,3,605.4275,1.0,1.0,1.0,1.0 -app_default,684,3,595.84642,0.0,0.0,0.0,0.0 -app_default,690,0,799.61146,0.0,0.0,0.0,0.0 -app_default,691,3,740.31371,0.63093,1.0,0.5,1.0 -app_default,692,1,682.43958,1.0,1.0,1.0,1.0 -app_default,693,1,772.58671,1.0,1.0,1.0,1.0 -app_default,700,1,698.99983,0.0,0.0,0.0,0.0 -app_default,702,2,704.21771,0.0,0.0,0.0,0.0 -app_default,715,3,781.39083,0.0,0.0,0.0,0.0 -app_default,716,2,827.23229,0.0,0.0,0.0,0.0 -app_default,718,0,768.04312,0.0,0.0,0.0,0.0 -app_default,721,1,780.13275,1.0,1.0,1.0,1.0 -app_default,723,1,410.21583,1.0,1.0,1.0,1.0 -app_default,727,2,743.2585,0.0,0.0,0.0,0.0 -app_default,728,3,708.44729,0.61315,0.5,1.0,1.0 -app_default,729,2,753.06217,1.0,1.0,1.0,1.0 -app_default,742,2,787.83425,1.0,1.0,1.0,1.0 -app_default,743,2,814.36083,1.0,1.0,1.0,1.0 -app_default,744,3,363.45696,0.63093,1.0,0.5,1.0 -app_default,756,3,723.89579,0.5,1.0,0.33333,1.0 -app_default,759,3,579.48396,1.0,1.0,1.0,1.0 -app_default,768,1,814.49242,0.0,0.0,0.0,0.0 -app_default,770,2,699.36254,1.0,1.0,1.0,1.0 -app_default,775,3,407.95508,0.0,0.0,0.0,0.0 -app_default,781,3,537.958,0.63093,1.0,0.5,1.0 -app_default,783,3,796.86204,0.0,0.0,0.0,0.0 -app_default,784,3,874.59608,1.0,1.0,1.0,1.0 -app_default,785,1,913.38913,0.0,0.0,0.0,0.0 -app_default,793,2,725.96292,1.0,1.0,1.0,1.0 -app_default,800,3,729.81962,0.63093,1.0,0.5,1.0 -app_default,805,3,571.85358,1.0,1.0,1.0,1.0 -app_default,808,1,745.03021,1.0,1.0,1.0,1.0 -app_default,811,3,784.61292,1.0,1.0,1.0,1.0 -app_default,814,2,760.47513,1.0,1.0,1.0,1.0 -app_default,820,0,706.38912,0.0,0.0,0.0,0.0 -app_default,821,0,880.93133,0.0,0.0,0.0,0.0 -app_default,823,3,742.058,1.0,1.0,1.0,1.0 -app_default,830,1,776.56558,0.0,0.0,0.0,0.0 -app_default,831,0,815.42025,0.0,0.0,0.0,0.0 -app_default,832,3,824.56333,1.0,1.0,1.0,1.0 -app_default,834,1,852.11958,0.0,0.0,0.0,0.0 -app_default,837,3,670.46383,0.63093,1.0,0.5,1.0 -app_default,839,3,717.87083,1.0,1.0,1.0,1.0 -app_default,845,3,467.56258,1.0,1.0,1.0,1.0 -app_default,847,2,620.88642,1.0,1.0,1.0,1.0 -app_default,852,3,665.52158,0.63093,1.0,0.5,1.0 -app_default,859,3,505.71846,0.63093,1.0,0.5,1.0 -app_default,870,2,612.82421,0.0,0.0,0.0,0.0 -app_default,873,3,861.677,0.0,0.0,0.0,0.0 -app_default,879,1,909.55642,1.0,1.0,1.0,1.0 -app_default,880,1,973.89579,1.0,1.0,1.0,1.0 -app_default,882,1,572.01079,1.0,1.0,1.0,1.0 -app_default,887,0,903.66167,0.0,0.0,0.0,0.0 -app_default,903,3,816.04487,1.0,1.0,1.0,1.0 -app_default,904,2,919.36633,1.0,1.0,1.0,1.0 -app_default,907,3,713.98417,1.0,1.0,1.0,1.0 -app_default,911,3,819.69567,1.0,1.0,1.0,1.0 -app_default,913,3,873.49112,0.0,0.0,0.0,0.0 -app_default,914,3,903.33104,0.0,0.0,0.0,0.0 -app_default,921,3,894.24017,1.0,1.0,1.0,1.0 -app_default,922,1,906.97462,1.0,1.0,1.0,1.0 -app_default,936,1,850.9535,1.0,1.0,1.0,1.0 -app_default,956,3,743.76733,1.0,1.0,1.0,1.0 -app_default,957,1,457.60063,1.0,1.0,1.0,1.0 -app_default,960,1,916.61479,1.0,1.0,1.0,1.0 -app_default,967,3,895.10279,0.91972,1.0,1.0,1.0 -app_default,971,3,466.50525,0.63668,0.5,1.0,1.0 -app_default,975,3,852.05087,0.0,0.0,0.0,0.0 -app_default,982,1,1046.89938,1.0,1.0,1.0,1.0 -app_default,985,3,817.98775,1.0,1.0,1.0,1.0 -app_default,993,3,695.41554,1.0,1.0,1.0,1.0 -app_default,1012,3,708.15521,1.0,1.0,1.0,1.0 -app_default,1014,1,883.168,1.0,1.0,1.0,1.0 -app_default,1019,2,783.92033,0.63093,1.0,0.5,1.0 -app_default,1020,2,760.8055,0.63093,1.0,0.5,1.0 -app_default,1021,1,764.42837,0.0,0.0,0.0,0.0 -app_default,1024,2,780.16996,1.0,1.0,1.0,1.0 -app_default,1029,3,885.79921,0.76536,0.66667,1.0,1.0 -app_default,1041,3,664.58283,0.0,0.0,0.0,0.0 -app_default,1049,0,764.43196,0.0,0.0,0.0,0.0 -app_default,1062,3,926.00329,1.0,1.0,1.0,1.0 -app_default,1086,1,930.66396,1.0,1.0,1.0,1.0 -app_default,1088,3,859.65596,1.0,1.0,1.0,1.0 -app_default,1089,2,670.68075,1.0,1.0,1.0,1.0 -app_default,1099,3,539.58233,0.0,0.0,0.0,0.0 -app_default,1100,3,495.8155,0.5,1.0,0.33333,1.0 -app_default,1104,2,565.04571,1.0,1.0,1.0,1.0 -app_default,1107,3,525.30162,1.0,1.0,1.0,1.0 -app_default,1110,0,743.51863,0.0,0.0,0.0,0.0 -app_default,1121,3,571.05683,1.0,1.0,1.0,1.0 -app_default,1130,3,574.50225,1.0,1.0,1.0,1.0 -app_default,1132,3,579.45783,0.61315,0.5,1.0,1.0 -app_default,1137,3,617.57725,1.0,1.0,1.0,1.0 -app_default,1140,0,542.29375,0.0,0.0,0.0,0.0 -app_default,1144,2,578.59062,1.0,1.0,1.0,1.0 -app_default,1146,1,708.45825,1.0,1.0,1.0,1.0 -app_default,1150,3,667.89108,1.0,1.0,1.0,1.0 -app_default,1163,2,526.11475,1.0,1.0,1.0,1.0 -app_default,1175,0,643.66792,0.0,0.0,0.0,0.0 -app_default,1179,1,637.13617,0.0,0.0,0.0,0.0 -app_default,1180,3,656.05108,0.0,0.0,0.0,0.0 -app_default,1185,1,588.37892,1.0,1.0,1.0,1.0 -app_default,1187,3,685.66883,1.0,1.0,1.0,1.0 -app_default,1191,0,821.88167,0.0,0.0,0.0,0.0 -app_default,1194,1,769.64971,1.0,1.0,1.0,1.0 -app_default,1196,0,730.60621,0.0,0.0,0.0,0.0 -app_default,1197,0,736.59813,0.0,0.0,0.0,0.0 -app_default,1199,2,646.26733,0.0,0.0,0.0,0.0 -app_default,1200,0,757.19967,0.0,0.0,0.0,0.0 -app_default,1202,3,579.84438,0.63093,1.0,0.5,1.0 -app_default,1204,1,724.14492,0.0,0.0,0.0,0.0 -app_default,1207,1,808.78025,1.0,1.0,1.0,1.0 -app_default,1213,3,540.08163,0.0,0.0,0.0,0.0 -app_default,1216,1,693.947,1.0,1.0,1.0,1.0 -app_default,1221,0,755.44562,0.0,0.0,0.0,0.0 -app_default,1225,2,718.55158,1.0,1.0,1.0,1.0 -app_default,1226,1,695.23721,0.0,0.0,0.0,0.0 -app_default,1232,1,567.91683,1.0,1.0,1.0,1.0 -app_default,1241,3,697.47492,1.0,1.0,1.0,1.0 -app_default,1245,0,708.46496,0.0,0.0,0.0,0.0 -app_default,1259,3,573.31092,1.0,1.0,1.0,1.0 -app_default,1262,3,341.16463,1.0,1.0,1.0,1.0 -app_default,1266,3,545.09392,0.5,1.0,0.33333,1.0 -app_default,1270,1,556.89196,1.0,1.0,1.0,1.0 -app_default,1271,1,547.14833,1.0,1.0,1.0,1.0 -app_default,1272,0,660.04,0.0,0.0,0.0,0.0 -app_default,1273,3,581.79471,1.0,1.0,1.0,1.0 -app_default,1274,3,466.90596,0.76536,0.66667,1.0,1.0 -app_default,1278,0,711.62471,0.0,0.0,0.0,0.0 -app_default,1279,1,669.28946,0.0,0.0,0.0,0.0 -app_default,1280,3,587.39379,0.0,0.0,0.0,0.0 -app_default,1281,1,600.80612,0.0,0.0,0.0,0.0 -app_default,1282,1,655.88812,1.0,1.0,1.0,1.0 -app_default,1290,1,637.06587,1.0,1.0,1.0,1.0 -app_default,1292,3,611.86692,0.63093,1.0,0.5,1.0 -app_default,1298,1,561.29708,1.0,1.0,1.0,1.0 -app_default,1303,0,653.83454,0.0,0.0,0.0,0.0 -app_default,1316,1,636.11296,0.0,0.0,0.0,0.0 -app_default,1319,3,593.17446,1.0,1.0,1.0,1.0 -app_default,1320,3,567.89046,1.0,1.0,1.0,1.0 -app_default,1332,3,631.60187,0.0,0.0,0.0,0.0 -app_default,1335,3,511.08204,1.0,1.0,1.0,1.0 -app_default,1336,3,539.93029,1.0,1.0,1.0,1.0 -app_default,1337,3,713.10454,1.0,1.0,1.0,1.0 -app_default,1339,1,611.36183,1.0,1.0,1.0,1.0 -app_default,1344,3,601.67967,0.63093,1.0,0.5,1.0 -app_default,1352,2,553.84667,1.0,1.0,1.0,1.0 -app_default,1359,3,535.67854,1.0,1.0,1.0,1.0 -app_default,1362,0,653.35379,0.0,0.0,0.0,0.0 -app_default,1363,0,546.2155,0.0,0.0,0.0,0.0 -app_default,1368,3,538.06521,0.0,0.0,0.0,0.0 -app_default,1370,2,553.25604,0.63093,1.0,0.5,1.0 -app_default,1379,3,596.92317,0.39038,0.25,1.0,1.0 -app_default,1382,3,590.72392,1.0,1.0,1.0,1.0 -app_default,1385,2,588.92204,1.0,1.0,1.0,1.0 -app_default,1389,3,630.72663,1.0,1.0,1.0,1.0 -app_default,1395,0,613.716,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1,3,661.42662,0.0,0.0,0.0,0.0 -app_default_no_cutoff,3,3,412.73037,1.0,1.0,1.0,1.0 -app_default_no_cutoff,5,3,692.40758,1.0,1.0,1.0,1.0 -app_default_no_cutoff,13,3,591.92913,0.0,0.0,0.0,0.0 -app_default_no_cutoff,36,3,614.27683,0.0,0.0,0.0,0.0 -app_default_no_cutoff,42,3,605.76646,1.0,1.0,1.0,1.0 -app_default_no_cutoff,48,3,688.8775,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,49,3,554.64387,1.0,1.0,1.0,1.0 -app_default_no_cutoff,50,3,410.83317,1.0,1.0,1.0,1.0 -app_default_no_cutoff,51,3,686.34754,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,53,3,666.61983,1.0,1.0,1.0,1.0 -app_default_no_cutoff,54,3,646.58879,0.0,0.0,0.0,0.0 -app_default_no_cutoff,56,3,659.64554,1.0,1.0,1.0,1.0 -app_default_no_cutoff,57,3,614.01013,1.0,1.0,1.0,1.0 -app_default_no_cutoff,70,3,714.82338,0.0,0.0,0.0,0.0 -app_default_no_cutoff,72,3,691.35904,1.0,1.0,1.0,1.0 -app_default_no_cutoff,75,3,522.97204,1.0,1.0,1.0,1.0 -app_default_no_cutoff,94,3,753.67462,0.0,0.0,0.0,0.0 -app_default_no_cutoff,99,3,725.78983,1.0,1.0,1.0,1.0 -app_default_no_cutoff,100,3,382.07196,1.0,1.0,1.0,1.0 -app_default_no_cutoff,113,3,492.82971,1.0,1.0,1.0,1.0 -app_default_no_cutoff,115,3,511.35883,1.0,1.0,1.0,1.0 -app_default_no_cutoff,118,3,508.09275,1.0,1.0,1.0,1.0 -app_default_no_cutoff,124,3,576.33842,1.0,1.0,1.0,1.0 -app_default_no_cutoff,127,3,605.493,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,128,3,741.06175,0.0,0.0,0.0,0.0 -app_default_no_cutoff,129,3,699.73596,1.0,1.0,1.0,1.0 -app_default_no_cutoff,130,3,612.16754,1.0,1.0,1.0,1.0 -app_default_no_cutoff,132,3,531.18413,0.0,0.0,0.0,0.0 -app_default_no_cutoff,133,3,609.04517,0.55315,0.4,1.0,1.0 -app_default_no_cutoff,137,3,674.72425,1.0,1.0,1.0,1.0 -app_default_no_cutoff,141,3,536.12446,0.91972,1.0,1.0,1.0 -app_default_no_cutoff,142,3,704.71667,1.0,1.0,1.0,1.0 -app_default_no_cutoff,143,3,610.44433,1.0,1.0,1.0,1.0 -app_default_no_cutoff,146,3,598.74771,1.0,1.0,1.0,1.0 -app_default_no_cutoff,148,3,500.76792,1.0,1.0,1.0,1.0 -app_default_no_cutoff,163,3,477.863,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,171,3,501.54221,1.0,1.0,1.0,1.0 -app_default_no_cutoff,179,3,616.68154,0.63668,0.5,1.0,1.0 -app_default_no_cutoff,180,3,605.97975,1.0,1.0,1.0,1.0 -app_default_no_cutoff,183,3,489.16271,1.0,1.0,1.0,1.0 -app_default_no_cutoff,185,3,711.17988,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,198,3,750.585,0.0,0.0,0.0,0.0 -app_default_no_cutoff,208,3,671.97237,1.0,1.0,1.0,1.0 -app_default_no_cutoff,212,3,641.60229,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,213,3,527.56321,0.0,0.0,0.0,0.0 -app_default_no_cutoff,216,3,651.08287,1.0,1.0,1.0,1.0 -app_default_no_cutoff,217,3,669.32375,1.0,1.0,1.0,1.0 -app_default_no_cutoff,218,3,574.71825,1.0,1.0,1.0,1.0 -app_default_no_cutoff,219,3,569.46962,1.0,1.0,1.0,1.0 -app_default_no_cutoff,230,3,609.60588,1.0,1.0,1.0,1.0 -app_default_no_cutoff,232,3,585.51392,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,233,3,464.41742,1.0,1.0,1.0,1.0 -app_default_no_cutoff,236,3,505.57183,1.0,1.0,1.0,1.0 -app_default_no_cutoff,237,3,382.62842,0.0,0.0,0.0,0.0 -app_default_no_cutoff,238,3,537.83208,0.0,0.0,0.0,0.0 -app_default_no_cutoff,239,3,432.74279,0.0,0.0,0.0,0.0 -app_default_no_cutoff,248,3,656.80879,1.0,1.0,1.0,1.0 -app_default_no_cutoff,249,3,695.05417,1.0,1.0,1.0,1.0 -app_default_no_cutoff,261,3,550.45325,0.38685,0.5,0.5,1.0 -app_default_no_cutoff,268,3,581.96612,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,269,3,437.79704,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,274,3,538.19538,1.0,1.0,1.0,1.0 -app_default_no_cutoff,275,3,588.76967,0.70392,0.66667,1.0,1.0 -app_default_no_cutoff,279,3,628.24067,1.0,1.0,1.0,1.0 -app_default_no_cutoff,294,3,478.12871,0.0,0.0,0.0,0.0 -app_default_no_cutoff,295,3,366.877,1.0,1.0,1.0,1.0 -app_default_no_cutoff,298,3,674.83246,1.0,1.0,1.0,1.0 -app_default_no_cutoff,300,3,686.10167,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,303,3,670.12071,0.0,0.0,0.0,0.0 -app_default_no_cutoff,312,3,641.30946,0.0,0.0,0.0,0.0 -app_default_no_cutoff,314,3,654.12512,0.0,0.0,0.0,0.0 -app_default_no_cutoff,324,3,640.54821,0.0,0.0,0.0,0.0 -app_default_no_cutoff,327,3,682.76237,1.0,1.0,1.0,1.0 -app_default_no_cutoff,338,3,580.7045,1.0,1.0,1.0,1.0 -app_default_no_cutoff,343,3,630.40079,0.30657,0.5,0.33333,1.0 -app_default_no_cutoff,350,3,543.01104,1.0,1.0,1.0,1.0 -app_default_no_cutoff,354,3,417.34137,1.0,1.0,1.0,1.0 -app_default_no_cutoff,362,3,633.80542,1.0,1.0,1.0,1.0 -app_default_no_cutoff,380,3,649.26708,1.0,1.0,1.0,1.0 -app_default_no_cutoff,384,3,649.81896,0.0,0.0,0.0,0.0 -app_default_no_cutoff,385,3,641.60812,0.38685,0.5,0.5,1.0 -app_default_no_cutoff,386,3,600.67646,1.0,1.0,1.0,1.0 -app_default_no_cutoff,388,3,656.5535,0.0,0.0,0.0,0.0 -app_default_no_cutoff,399,3,541.29325,1.0,1.0,1.0,1.0 -app_default_no_cutoff,410,3,593.147,1.0,1.0,1.0,1.0 -app_default_no_cutoff,411,3,558.16821,1.0,1.0,1.0,1.0 -app_default_no_cutoff,415,3,559.87658,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,421,3,763.41313,0.0,0.0,0.0,0.0 -app_default_no_cutoff,431,3,525.76725,0.0,0.0,0.0,0.0 -app_default_no_cutoff,436,3,590.65533,1.0,1.0,1.0,1.0 -app_default_no_cutoff,437,3,613.55746,0.0,0.0,0.0,0.0 -app_default_no_cutoff,439,3,673.24492,1.0,1.0,1.0,1.0 -app_default_no_cutoff,440,3,713.1925,1.0,1.0,1.0,1.0 -app_default_no_cutoff,443,3,560.24858,1.0,1.0,1.0,1.0 -app_default_no_cutoff,452,3,675.06437,0.38685,0.5,0.5,1.0 -app_default_no_cutoff,475,3,569.73133,0.0,0.0,0.0,0.0 -app_default_no_cutoff,478,3,691.6615,1.0,1.0,1.0,1.0 -app_default_no_cutoff,491,3,688.55929,1.0,1.0,1.0,1.0 -app_default_no_cutoff,501,3,610.68758,1.0,1.0,1.0,1.0 -app_default_no_cutoff,502,3,759.93779,0.0,0.0,0.0,0.0 -app_default_no_cutoff,507,3,646.48187,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,508,3,570.922,0.0,0.0,0.0,0.0 -app_default_no_cutoff,513,3,651.39821,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,514,3,675.34167,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,516,3,525.24021,1.0,1.0,1.0,1.0 -app_default_no_cutoff,517,3,673.07658,0.0,0.0,0.0,0.0 -app_default_no_cutoff,521,3,536.17337,1.0,1.0,1.0,1.0 -app_default_no_cutoff,525,3,495.46463,1.0,1.0,1.0,1.0 -app_default_no_cutoff,527,3,663.82708,0.0,0.0,0.0,0.0 -app_default_no_cutoff,528,3,690.07542,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,532,3,640.88021,1.0,1.0,1.0,1.0 -app_default_no_cutoff,533,3,657.0045,1.0,1.0,1.0,1.0 -app_default_no_cutoff,535,3,690.3485,0.0,0.0,0.0,0.0 -app_default_no_cutoff,536,3,700.90629,1.0,1.0,1.0,1.0 -app_default_no_cutoff,539,3,571.41883,1.0,1.0,1.0,1.0 -app_default_no_cutoff,540,3,568.50417,0.30657,0.5,0.33333,1.0 -app_default_no_cutoff,544,3,643.2575,0.0,0.0,0.0,0.0 -app_default_no_cutoff,549,3,679.53804,1.0,1.0,1.0,1.0 -app_default_no_cutoff,551,3,670.201,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,552,3,419.34075,1.0,1.0,1.0,1.0 -app_default_no_cutoff,554,3,421.16383,0.0,0.0,0.0,0.0 -app_default_no_cutoff,560,3,652.08975,0.0,0.0,0.0,0.0 -app_default_no_cutoff,569,3,650.15292,1.0,1.0,1.0,1.0 -app_default_no_cutoff,575,3,700.678,1.0,1.0,1.0,1.0 -app_default_no_cutoff,577,3,673.2165,0.0,0.0,0.0,0.0 -app_default_no_cutoff,578,3,708.07217,1.0,1.0,1.0,1.0 -app_default_no_cutoff,587,3,715.93071,0.0,0.0,0.0,0.0 -app_default_no_cutoff,589,3,596.84192,1.0,1.0,1.0,1.0 -app_default_no_cutoff,593,3,586.42108,1.0,1.0,1.0,1.0 -app_default_no_cutoff,597,3,421.77946,0.70392,0.66667,1.0,1.0 -app_default_no_cutoff,598,3,493.85175,0.0,0.0,0.0,0.0 -app_default_no_cutoff,613,3,670.4765,1.0,1.0,1.0,1.0 -app_default_no_cutoff,619,3,693.85325,0.61315,0.5,1.0,1.0 -app_default_no_cutoff,623,3,577.43721,0.0,0.0,0.0,0.0 -app_default_no_cutoff,628,3,640.36154,1.0,1.0,1.0,1.0 -app_default_no_cutoff,636,3,640.24075,1.0,1.0,1.0,1.0 -app_default_no_cutoff,637,3,695.04525,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,641,3,642.77437,0.61315,0.5,1.0,1.0 -app_default_no_cutoff,644,3,689.08087,1.0,1.0,1.0,1.0 -app_default_no_cutoff,649,3,689.08433,1.0,1.0,1.0,1.0 -app_default_no_cutoff,659,3,731.576,0.0,0.0,0.0,0.0 -app_default_no_cutoff,660,3,765.07546,0.0,0.0,0.0,0.0 -app_default_no_cutoff,674,3,580.72354,1.0,1.0,1.0,1.0 -app_default_no_cutoff,684,3,584.21271,0.0,0.0,0.0,0.0 -app_default_no_cutoff,690,3,746.4585,0.0,0.0,0.0,0.0 -app_default_no_cutoff,691,3,619.90546,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,692,3,598.23892,1.0,1.0,1.0,1.0 -app_default_no_cutoff,693,3,683.77796,1.0,1.0,1.0,1.0 -app_default_no_cutoff,700,3,587.43129,0.0,0.0,0.0,0.0 -app_default_no_cutoff,702,3,577.62275,0.0,0.0,0.0,0.0 -app_default_no_cutoff,715,3,707.73587,0.0,0.0,0.0,0.0 -app_default_no_cutoff,716,3,727.84879,0.0,0.0,0.0,0.0 -app_default_no_cutoff,718,3,601.61988,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,721,3,729.714,1.0,1.0,1.0,1.0 -app_default_no_cutoff,723,3,425.99242,1.0,1.0,1.0,1.0 -app_default_no_cutoff,727,3,677.70021,0.0,0.0,0.0,0.0 -app_default_no_cutoff,728,3,672.32308,0.61315,0.5,1.0,1.0 -app_default_no_cutoff,729,3,661.08858,1.0,1.0,1.0,1.0 -app_default_no_cutoff,742,3,678.17129,1.0,1.0,1.0,1.0 -app_default_no_cutoff,743,3,757.14883,1.0,1.0,1.0,1.0 -app_default_no_cutoff,744,3,388.81188,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,756,3,685.86521,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,759,3,568.30342,1.0,1.0,1.0,1.0 -app_default_no_cutoff,768,3,644.45446,0.0,0.0,0.0,0.0 -app_default_no_cutoff,770,3,738.96208,1.0,1.0,1.0,1.0 -app_default_no_cutoff,775,3,426.87179,0.0,0.0,0.0,0.0 -app_default_no_cutoff,781,3,535.95058,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,783,3,629.36896,0.0,0.0,0.0,0.0 -app_default_no_cutoff,784,3,726.54204,1.0,1.0,1.0,1.0 -app_default_no_cutoff,785,3,685.99963,0.0,0.0,0.0,0.0 -app_default_no_cutoff,793,3,695.215,1.0,1.0,1.0,1.0 -app_default_no_cutoff,800,3,654.8685,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,805,3,530.95729,1.0,1.0,1.0,1.0 -app_default_no_cutoff,808,3,747.50567,1.0,1.0,1.0,1.0 -app_default_no_cutoff,811,3,672.61304,1.0,1.0,1.0,1.0 -app_default_no_cutoff,814,3,705.85946,1.0,1.0,1.0,1.0 -app_default_no_cutoff,820,3,677.4335,0.0,0.0,0.0,0.0 -app_default_no_cutoff,821,3,781.443,0.0,0.0,0.0,0.0 -app_default_no_cutoff,823,3,605.64175,1.0,1.0,1.0,1.0 -app_default_no_cutoff,830,3,605.78179,0.0,0.0,0.0,0.0 -app_default_no_cutoff,831,3,603.52912,0.0,0.0,0.0,0.0 -app_default_no_cutoff,832,3,692.61454,1.0,1.0,1.0,1.0 -app_default_no_cutoff,834,3,732.00237,0.0,0.0,0.0,0.0 -app_default_no_cutoff,837,3,687.50408,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,839,3,670.50875,1.0,1.0,1.0,1.0 -app_default_no_cutoff,845,3,420.87879,1.0,1.0,1.0,1.0 -app_default_no_cutoff,847,3,521.32863,1.0,1.0,1.0,1.0 -app_default_no_cutoff,852,3,656.36475,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,859,3,478.05458,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,870,3,629.67396,0.0,0.0,0.0,0.0 -app_default_no_cutoff,873,3,691.17058,0.0,0.0,0.0,0.0 -app_default_no_cutoff,879,3,749.61833,1.0,1.0,1.0,1.0 -app_default_no_cutoff,880,3,633.70242,1.0,1.0,1.0,1.0 -app_default_no_cutoff,882,3,651.36875,1.0,1.0,1.0,1.0 -app_default_no_cutoff,887,3,703.08296,0.0,0.0,0.0,0.0 -app_default_no_cutoff,903,3,674.875,1.0,1.0,1.0,1.0 -app_default_no_cutoff,904,3,738.79071,1.0,1.0,1.0,1.0 -app_default_no_cutoff,907,3,608.49983,1.0,1.0,1.0,1.0 -app_default_no_cutoff,911,3,675.12696,1.0,1.0,1.0,1.0 -app_default_no_cutoff,913,3,827.28,0.0,0.0,0.0,0.0 -app_default_no_cutoff,914,3,825.54763,0.0,0.0,0.0,0.0 -app_default_no_cutoff,921,3,707.12796,1.0,1.0,1.0,1.0 -app_default_no_cutoff,922,3,696.73108,1.0,1.0,1.0,1.0 -app_default_no_cutoff,936,3,787.27483,1.0,1.0,1.0,1.0 -app_default_no_cutoff,956,3,689.51304,1.0,1.0,1.0,1.0 -app_default_no_cutoff,957,3,464.17588,1.0,1.0,1.0,1.0 -app_default_no_cutoff,960,3,697.03742,1.0,1.0,1.0,1.0 -app_default_no_cutoff,967,3,685.03842,0.91972,1.0,1.0,1.0 -app_default_no_cutoff,971,3,453.58792,0.63668,0.5,1.0,1.0 -app_default_no_cutoff,975,3,703.25463,0.0,0.0,0.0,0.0 -app_default_no_cutoff,982,3,752.03838,1.0,1.0,1.0,1.0 -app_default_no_cutoff,985,3,580.68954,1.0,1.0,1.0,1.0 -app_default_no_cutoff,993,3,655.42113,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1012,3,437.208,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1014,3,689.09217,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1019,3,708.11596,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1020,3,636.65413,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1021,3,641.75283,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1024,3,767.49383,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1029,3,637.22663,0.76536,0.66667,1.0,1.0 -app_default_no_cutoff,1041,3,566.979,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1049,3,675.98608,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1062,3,693.94342,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1086,3,706.56612,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1088,3,788.46125,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1089,3,572.20083,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1099,3,599.21254,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1100,3,604.13821,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,1104,3,726.58854,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1107,3,541.86913,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1110,3,817.30971,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1121,3,617.13517,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1130,3,729.17558,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1132,3,703.94196,0.61315,0.5,1.0,1.0 -app_default_no_cutoff,1137,3,767.5925,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1140,3,625.3885,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1144,3,691.47558,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1146,3,722.41179,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1150,3,762.40363,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1163,3,533.13429,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1175,3,743.61708,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1179,3,764.321,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1180,3,760.63167,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1185,3,642.74337,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1187,3,759.79654,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1191,3,798.82108,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1194,3,749.88425,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1196,3,792.60512,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1197,3,825.73429,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1199,3,681.10696,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1200,3,864.36621,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1202,3,670.10971,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1204,3,721.57904,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1207,3,747.98883,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1213,3,620.71879,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1216,3,778.00133,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1221,3,843.88625,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1225,3,767.41279,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1226,3,810.311,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1232,3,693.73388,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1241,3,832.90171,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1245,3,819.10758,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1259,3,594.73804,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1262,3,356.60171,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1266,3,531.76583,0.5,1.0,0.33333,1.0 -app_default_no_cutoff,1270,3,652.63263,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1271,3,631.33079,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1272,3,802.23633,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1273,3,690.74162,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1274,3,461.39071,0.76536,0.66667,1.0,1.0 -app_default_no_cutoff,1278,3,804.65137,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1279,3,653.21575,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1280,3,698.22313,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1281,3,713.03458,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1282,3,815.88125,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1290,3,750.12575,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1292,3,738.55246,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1298,3,617.0095,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1303,3,807.25171,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1316,3,752.78592,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1319,3,730.94258,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1320,3,666.38458,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1332,3,712.42762,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1335,3,584.11512,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1336,3,626.24792,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1337,3,803.47167,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1339,3,801.06663,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1344,3,715.76254,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1352,3,712.08237,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1359,3,629.71896,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1362,3,796.93971,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1363,3,580.59117,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1368,3,602.16313,0.0,0.0,0.0,0.0 -app_default_no_cutoff,1370,3,546.15554,0.63093,1.0,0.5,1.0 -app_default_no_cutoff,1379,3,707.69267,0.39038,0.25,1.0,1.0 -app_default_no_cutoff,1382,3,722.6655,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1385,3,607.096,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1389,3,671.60225,1.0,1.0,1.0,1.0 -app_default_no_cutoff,1395,3,722.11004,1.0,1.0,1.0,1.0 +dense,1,10,11.07037,0.38685,1.0,0.2,0.0 +dense,3,10,10.64308,0.5,1.0,0.33333,1.0 +dense,5,10,10.33983,0.63093,1.0,0.5,1.0 +dense,13,10,9.94171,0.0,0.0,0.0,0.0 +dense,36,10,9.59754,0.49115,1.0,0.33333,1.0 +dense,42,10,10.50717,0.63093,1.0,0.5,1.0 +dense,48,10,9.6585,0.0,0.0,0.0,0.0 +dense,49,10,10.56937,1.0,1.0,1.0,1.0 +dense,50,10,9.53242,1.0,1.0,1.0,1.0 +dense,51,10,9.5785,1.0,1.0,1.0,1.0 +dense,53,10,10.32088,1.0,1.0,1.0,1.0 +dense,54,10,9.72796,1.0,1.0,1.0,1.0 +dense,56,10,8.68096,1.0,1.0,1.0,1.0 +dense,57,10,9.74746,1.0,1.0,1.0,1.0 +dense,70,10,9.69479,0.0,0.0,0.0,0.0 +dense,72,10,10.23958,1.0,1.0,1.0,1.0 +dense,75,10,9.20446,0.63093,1.0,0.5,1.0 +dense,94,10,10.60225,0.5,1.0,0.33333,1.0 +dense,99,10,10.32913,0.63093,1.0,0.5,1.0 +dense,100,10,9.65554,0.5,1.0,0.33333,1.0 +dense,113,10,9.17125,1.0,1.0,1.0,1.0 +dense,115,10,10.14546,1.0,1.0,1.0,1.0 +dense,118,10,9.454,0.63093,1.0,0.5,1.0 +dense,124,10,9.76971,1.0,1.0,1.0,1.0 +dense,127,10,10.40829,1.0,1.0,1.0,1.0 +dense,128,10,10.3375,0.0,0.0,0.0,0.0 +dense,129,10,9.94975,1.0,1.0,1.0,1.0 +dense,130,10,9.30992,1.0,1.0,1.0,1.0 +dense,132,10,10.53967,0.0,0.0,0.0,0.0 +dense,133,10,10.54812,0.63995,0.6,1.0,1.0 +dense,137,10,10.05629,1.0,1.0,1.0,1.0 +dense,141,10,10.77075,0.91972,1.0,1.0,1.0 +dense,142,10,9.63917,1.0,1.0,1.0,1.0 +dense,143,10,10.2045,1.0,1.0,1.0,1.0 +dense,146,10,10.13167,1.0,1.0,1.0,1.0 +dense,148,10,8.52404,0.63093,1.0,0.5,1.0 +dense,163,10,9.52838,1.0,1.0,1.0,1.0 +dense,171,10,9.91663,1.0,1.0,1.0,1.0 +dense,179,10,9.89225,0.95583,1.0,1.0,1.0 +dense,180,10,10.67488,1.0,1.0,1.0,1.0 +dense,183,10,9.47367,1.0,1.0,1.0,1.0 +dense,185,10,9.77012,1.0,1.0,1.0,1.0 +dense,198,10,9.97654,0.0,0.0,0.0,0.0 +dense,208,10,9.45329,0.5,1.0,0.33333,1.0 +dense,212,10,9.42954,0.35621,1.0,0.16667,0.0 +dense,213,10,10.13221,0.38685,1.0,0.2,0.0 +dense,216,10,10.20754,1.0,1.0,1.0,1.0 +dense,217,10,10.35462,1.0,1.0,1.0,1.0 +dense,218,10,9.57779,1.0,1.0,1.0,1.0 +dense,219,10,10.49421,1.0,1.0,1.0,1.0 +dense,230,10,9.39321,1.0,1.0,1.0,1.0 +dense,232,10,9.95229,1.0,1.0,1.0,1.0 +dense,233,10,10.692,1.0,1.0,1.0,1.0 +dense,236,10,9.98892,1.0,1.0,1.0,1.0 +dense,237,10,9.46442,1.0,1.0,1.0,1.0 +dense,238,10,9.58058,0.63093,1.0,0.5,1.0 +dense,239,10,9.80054,0.0,0.0,0.0,0.0 +dense,248,10,10.11283,0.5,1.0,0.33333,1.0 +dense,249,10,9.81438,0.5,1.0,0.33333,1.0 +dense,261,10,9.85887,0.20438,0.5,0.14286,0.0 +dense,268,10,8.48292,1.0,1.0,1.0,1.0 +dense,269,10,7.97958,1.0,1.0,1.0,1.0 +dense,274,10,10.50612,1.0,1.0,1.0,1.0 +dense,275,10,9.98058,0.67973,1.0,0.5,1.0 +dense,279,10,9.35721,1.0,1.0,1.0,1.0 +dense,294,10,9.00154,0.0,0.0,0.0,0.0 +dense,295,10,10.97629,0.5,1.0,0.33333,1.0 +dense,298,10,9.36558,1.0,1.0,1.0,1.0 +dense,300,10,9.97504,0.43068,1.0,0.25,0.0 +dense,303,10,9.38696,0.0,0.0,0.0,0.0 +dense,312,10,9.832,0.0,0.0,0.0,0.0 +dense,314,10,9.64954,0.0,0.0,0.0,0.0 +dense,324,10,9.96304,0.63093,1.0,0.5,1.0 +dense,327,10,10.01871,1.0,1.0,1.0,1.0 +dense,338,10,10.40313,0.63093,1.0,0.5,1.0 +dense,343,10,8.63067,0.83155,1.0,1.0,1.0 +dense,350,10,9.16671,1.0,1.0,1.0,1.0 +dense,354,10,8.90642,1.0,1.0,1.0,1.0 +dense,362,10,10.25233,1.0,1.0,1.0,1.0 +dense,380,10,9.77529,1.0,1.0,1.0,1.0 +dense,384,10,9.51863,0.0,0.0,0.0,0.0 +dense,385,10,9.23929,0.69343,1.0,0.5,1.0 +dense,386,10,9.60517,1.0,1.0,1.0,1.0 +dense,388,10,10.08346,0.63093,1.0,0.5,1.0 +dense,399,10,9.89254,1.0,1.0,1.0,1.0 +dense,410,10,9.87442,1.0,1.0,1.0,1.0 +dense,411,10,9.54954,1.0,1.0,1.0,1.0 +dense,415,10,9.77617,0.0,0.0,0.0,0.0 +dense,421,10,9.32921,0.0,0.0,0.0,0.0 +dense,431,10,9.20437,0.0,0.0,0.0,0.0 +dense,436,10,9.48158,1.0,1.0,1.0,1.0 +dense,437,10,8.34587,0.0,0.0,0.0,0.0 +dense,439,10,9.14283,1.0,1.0,1.0,1.0 +dense,440,10,9.58183,1.0,1.0,1.0,1.0 +dense,443,10,8.72017,1.0,1.0,1.0,1.0 +dense,452,10,9.95946,0.17724,0.5,0.1,0.0 +dense,475,10,10.11917,0.0,0.0,0.0,0.0 +dense,478,10,10.63638,1.0,1.0,1.0,1.0 +dense,491,10,9.58758,1.0,1.0,1.0,1.0 +dense,501,10,10.253,1.0,1.0,1.0,1.0 +dense,502,10,9.42462,0.0,0.0,0.0,0.0 +dense,507,10,9.28042,0.63093,1.0,0.5,1.0 +dense,508,10,10.41842,0.0,0.0,0.0,0.0 +dense,513,10,10.46238,0.63093,1.0,0.5,1.0 +dense,514,10,10.30425,1.0,1.0,1.0,1.0 +dense,516,10,8.90221,1.0,1.0,1.0,1.0 +dense,517,10,9.72679,0.0,0.0,0.0,0.0 +dense,521,10,9.55167,1.0,1.0,1.0,1.0 +dense,525,10,9.34837,1.0,1.0,1.0,1.0 +dense,527,10,10.26146,0.0,0.0,0.0,0.0 +dense,528,10,10.38942,1.0,1.0,1.0,1.0 +dense,532,10,10.02217,1.0,1.0,1.0,1.0 +dense,533,10,9.29967,1.0,1.0,1.0,1.0 +dense,535,10,10.14021,0.43068,1.0,0.25,0.0 +dense,536,10,10.85663,1.0,1.0,1.0,1.0 +dense,539,10,10.07225,1.0,1.0,1.0,1.0 +dense,540,10,9.67879,0.26407,0.5,0.25,0.0 +dense,544,10,9.89962,0.0,0.0,0.0,0.0 +dense,549,10,11.15487,1.0,1.0,1.0,1.0 +dense,551,10,10.27258,0.63093,1.0,0.5,1.0 +dense,552,10,9.63508,1.0,1.0,1.0,1.0 +dense,554,10,9.02937,0.0,0.0,0.0,0.0 +dense,560,10,9.90404,0.0,0.0,0.0,0.0 +dense,569,10,9.8985,0.43068,1.0,0.25,0.0 +dense,575,10,9.96233,0.5,1.0,0.33333,1.0 +dense,577,10,10.07817,0.31546,1.0,0.125,0.0 +dense,578,10,9.40042,1.0,1.0,1.0,1.0 +dense,587,10,9.81775,0.0,0.0,0.0,0.0 +dense,589,10,9.64579,1.0,1.0,1.0,1.0 +dense,593,10,9.46079,1.0,1.0,1.0,1.0 +dense,597,10,8.34421,0.79287,1.0,1.0,1.0 +dense,598,10,9.87088,0.33333,1.0,0.14286,0.0 +dense,613,10,9.04392,1.0,1.0,1.0,1.0 +dense,619,10,10.41462,0.0,0.0,0.0,0.0 +dense,623,10,10.00454,0.0,0.0,0.0,0.0 +dense,628,10,10.02925,0.43068,1.0,0.25,0.0 +dense,636,10,10.28321,1.0,1.0,1.0,1.0 +dense,637,10,9.46379,0.43068,1.0,0.25,0.0 +dense,641,10,9.87358,0.83155,1.0,1.0,1.0 +dense,644,10,8.37725,0.0,0.0,0.0,0.0 +dense,649,10,10.31317,1.0,1.0,1.0,1.0 +dense,659,10,9.34929,0.43068,1.0,0.25,0.0 +dense,660,10,10.42571,0.0,0.0,0.0,0.0 +dense,674,10,10.3815,0.33333,1.0,0.14286,0.0 +dense,684,10,9.13604,1.0,1.0,1.0,1.0 +dense,690,10,9.50154,0.0,0.0,0.0,0.0 +dense,691,10,9.04725,0.43068,1.0,0.25,0.0 +dense,692,10,10.51742,1.0,1.0,1.0,1.0 +dense,693,10,10.36388,1.0,1.0,1.0,1.0 +dense,700,10,9.84163,0.63093,1.0,0.5,1.0 +dense,702,10,9.58433,0.63093,1.0,0.5,1.0 +dense,715,10,10.38392,0.0,0.0,0.0,0.0 +dense,716,10,10.14725,0.0,0.0,0.0,0.0 +dense,718,10,9.82704,1.0,1.0,1.0,1.0 +dense,721,10,8.74363,1.0,1.0,1.0,1.0 +dense,723,10,9.45592,1.0,1.0,1.0,1.0 +dense,727,10,8.87312,0.43068,1.0,0.25,0.0 +dense,728,10,9.53321,0.65092,1.0,0.5,1.0 +dense,729,10,8.85087,1.0,1.0,1.0,1.0 +dense,742,10,10.03642,0.30103,1.0,0.11111,0.0 +dense,743,10,10.19287,0.0,0.0,0.0,0.0 +dense,744,10,9.55746,0.5,1.0,0.33333,1.0 +dense,756,10,9.33975,0.63093,1.0,0.5,1.0 +dense,759,10,9.00208,1.0,1.0,1.0,1.0 +dense,768,10,10.45996,0.0,0.0,0.0,0.0 +dense,770,10,10.22738,1.0,1.0,1.0,1.0 +dense,775,10,9.97263,0.0,0.0,0.0,0.0 +dense,781,10,8.93396,0.63093,1.0,0.5,1.0 +dense,783,10,9.98967,0.38685,1.0,0.2,0.0 +dense,784,10,9.32021,0.63093,1.0,0.5,1.0 +dense,785,10,8.98333,0.0,0.0,0.0,0.0 +dense,793,10,9.41154,1.0,1.0,1.0,1.0 +dense,800,10,9.63242,0.63093,1.0,0.5,1.0 +dense,805,10,9.98237,1.0,1.0,1.0,1.0 +dense,808,10,10.18771,1.0,1.0,1.0,1.0 +dense,811,10,9.58542,1.0,1.0,1.0,1.0 +dense,814,10,10.42546,1.0,1.0,1.0,1.0 +dense,820,10,9.59833,0.0,0.0,0.0,0.0 +dense,821,10,10.034,0.0,0.0,0.0,0.0 +dense,823,10,9.96896,1.0,1.0,1.0,1.0 +dense,830,10,9.78283,0.0,0.0,0.0,0.0 +dense,831,10,9.666,0.0,0.0,0.0,0.0 +dense,832,10,9.7675,1.0,1.0,1.0,1.0 +dense,834,10,8.98167,0.0,0.0,0.0,0.0 +dense,837,10,10.04292,0.63093,1.0,0.5,1.0 +dense,839,10,9.52158,0.5,1.0,0.33333,1.0 +dense,845,10,9.39521,0.43068,1.0,0.25,0.0 +dense,847,10,9.04329,1.0,1.0,1.0,1.0 +dense,852,10,10.00408,0.63093,1.0,0.5,1.0 +dense,859,10,9.88625,0.43068,1.0,0.25,0.0 +dense,870,10,8.13292,0.0,0.0,0.0,0.0 +dense,873,10,7.84596,0.26762,0.4,0.33333,1.0 +dense,879,10,9.90233,1.0,1.0,1.0,1.0 +dense,880,10,10.54904,0.63093,1.0,0.5,1.0 +dense,882,10,9.9595,1.0,1.0,1.0,1.0 +dense,887,10,10.20008,0.0,0.0,0.0,0.0 +dense,903,10,9.59117,1.0,1.0,1.0,1.0 +dense,904,10,9.82804,1.0,1.0,1.0,1.0 +dense,907,10,9.26563,1.0,1.0,1.0,1.0 +dense,911,10,9.27821,1.0,1.0,1.0,1.0 +dense,913,10,10.13658,0.31546,1.0,0.125,0.0 +dense,914,10,9.88171,0.43068,1.0,0.25,0.0 +dense,921,10,9.01221,1.0,1.0,1.0,1.0 +dense,922,10,9.53817,1.0,1.0,1.0,1.0 +dense,936,10,10.1725,1.0,1.0,1.0,1.0 +dense,956,10,10.6065,1.0,1.0,1.0,1.0 +dense,957,10,9.88796,0.5,1.0,0.33333,1.0 +dense,960,10,9.52058,1.0,1.0,1.0,1.0 +dense,967,10,9.46987,1.0,1.0,1.0,1.0 +dense,971,10,9.2905,0.73157,1.0,0.5,1.0 +dense,975,10,10.10192,0.0,0.0,0.0,0.0 +dense,982,10,9.38671,1.0,1.0,1.0,1.0 +dense,985,10,8.497,1.0,1.0,1.0,1.0 +dense,993,10,9.41875,0.63093,1.0,0.5,1.0 +dense,1012,10,10.63658,1.0,1.0,1.0,1.0 +dense,1014,10,9.65996,1.0,1.0,1.0,1.0 +dense,1019,10,9.68533,0.43068,1.0,0.25,0.0 +dense,1020,10,8.95187,0.63093,1.0,0.5,1.0 +dense,1021,10,8.87192,0.63093,1.0,0.5,1.0 +dense,1024,10,10.65004,0.0,0.0,0.0,0.0 +dense,1029,10,9.85692,0.96747,1.0,1.0,1.0 +dense,1041,10,10.00004,0.0,0.0,0.0,0.0 +dense,1049,10,10.36417,0.0,0.0,0.0,0.0 +dense,1062,10,9.89896,0.63093,1.0,0.5,1.0 +dense,1086,10,9.62812,1.0,1.0,1.0,1.0 +dense,1088,10,9.73737,1.0,1.0,1.0,1.0 +dense,1089,10,9.78667,1.0,1.0,1.0,1.0 +dense,1099,10,8.45875,0.5,1.0,0.33333,1.0 +dense,1100,10,8.45238,0.43068,1.0,0.25,0.0 +dense,1104,10,10.00788,1.0,1.0,1.0,1.0 +dense,1107,10,10.36625,0.5,1.0,0.33333,1.0 +dense,1110,10,10.39008,0.0,0.0,0.0,0.0 +dense,1121,10,9.83758,1.0,1.0,1.0,1.0 +dense,1130,10,9.20721,1.0,1.0,1.0,1.0 +dense,1132,10,9.62796,0.2372,0.5,0.2,0.0 +dense,1137,10,9.83167,1.0,1.0,1.0,1.0 +dense,1140,10,10.4755,1.0,1.0,1.0,1.0 +dense,1144,10,9.9035,1.0,1.0,1.0,1.0 +dense,1146,10,9.69762,1.0,1.0,1.0,1.0 +dense,1150,10,9.06608,1.0,1.0,1.0,1.0 +dense,1163,10,10.51996,1.0,1.0,1.0,1.0 +dense,1175,10,10.73862,0.0,0.0,0.0,0.0 +dense,1179,10,10.02033,0.0,0.0,0.0,0.0 +dense,1180,10,9.12167,0.0,0.0,0.0,0.0 +dense,1185,10,10.65438,0.5,1.0,0.33333,1.0 +dense,1187,10,9.72917,0.63093,1.0,0.5,1.0 +dense,1191,10,9.40075,0.63093,1.0,0.5,1.0 +dense,1194,10,8.94754,1.0,1.0,1.0,1.0 +dense,1196,10,10.24087,0.38685,1.0,0.2,0.0 +dense,1197,10,9.76767,0.38685,1.0,0.2,0.0 +dense,1199,10,9.22717,0.0,0.0,0.0,0.0 +dense,1200,10,11.14775,0.0,0.0,0.0,0.0 +dense,1202,10,9.44308,0.63093,1.0,0.5,1.0 +dense,1204,10,8.64121,1.0,1.0,1.0,1.0 +dense,1207,10,10.23125,1.0,1.0,1.0,1.0 +dense,1213,10,10.78487,0.0,0.0,0.0,0.0 +dense,1216,10,11.46308,1.0,1.0,1.0,1.0 +dense,1221,10,10.44375,0.31546,1.0,0.125,0.0 +dense,1225,10,10.53971,1.0,1.0,1.0,1.0 +dense,1226,10,9.67821,0.0,0.0,0.0,0.0 +dense,1232,10,8.96417,1.0,1.0,1.0,1.0 +dense,1241,10,9.21446,1.0,1.0,1.0,1.0 +dense,1245,10,9.84579,0.30103,1.0,0.11111,0.0 +dense,1259,10,10.77667,0.63093,1.0,0.5,1.0 +dense,1262,10,10.19096,1.0,1.0,1.0,1.0 +dense,1266,10,9.82729,0.38685,1.0,0.2,0.0 +dense,1270,10,11.81133,1.0,1.0,1.0,1.0 +dense,1271,10,8.90512,1.0,1.0,1.0,1.0 +dense,1272,10,9.07292,0.0,0.0,0.0,0.0 +dense,1273,10,10.27525,1.0,1.0,1.0,1.0 +dense,1274,10,10.366,0.9469,1.0,1.0,1.0 +dense,1278,10,9.24513,1.0,1.0,1.0,1.0 +dense,1279,10,8.54583,1.0,1.0,1.0,1.0 +dense,1280,10,9.58587,0.0,0.0,0.0,0.0 +dense,1281,10,9.83808,0.31546,1.0,0.125,0.0 +dense,1282,10,9.62025,1.0,1.0,1.0,1.0 +dense,1290,10,9.61471,1.0,1.0,1.0,1.0 +dense,1292,10,10.15104,1.0,1.0,1.0,1.0 +dense,1298,10,10.19892,1.0,1.0,1.0,1.0 +dense,1303,10,9.65079,1.0,1.0,1.0,1.0 +dense,1316,10,9.86663,0.0,0.0,0.0,0.0 +dense,1319,10,10.087,1.0,1.0,1.0,1.0 +dense,1320,10,9.18621,1.0,1.0,1.0,1.0 +dense,1332,10,10.46483,0.0,0.0,0.0,0.0 +dense,1335,10,9.64404,1.0,1.0,1.0,1.0 +dense,1336,10,9.45446,1.0,1.0,1.0,1.0 +dense,1337,10,9.02829,0.28906,1.0,0.1,0.0 +dense,1339,10,11.55104,1.0,1.0,1.0,1.0 +dense,1344,10,10.11238,0.63093,1.0,0.5,1.0 +dense,1352,10,9.3555,1.0,1.0,1.0,1.0 +dense,1359,10,9.52521,1.0,1.0,1.0,1.0 +dense,1362,10,10.28083,0.0,0.0,0.0,0.0 +dense,1363,10,10.31967,0.0,0.0,0.0,0.0 +dense,1368,10,9.61383,0.0,0.0,0.0,0.0 +dense,1370,10,9.96567,0.5,1.0,0.33333,1.0 +dense,1379,10,10.12071,0.95583,1.0,1.0,1.0 +dense,1382,10,10.02496,1.0,1.0,1.0,1.0 +dense,1385,10,9.63575,0.63093,1.0,0.5,1.0 +dense,1389,10,10.18604,1.0,1.0,1.0,1.0 +dense,1395,10,10.12688,0.5,1.0,0.33333,1.0 +bm25,1,10,6.51933,0.0,0.0,0.0,0.0 +bm25,3,10,17.55092,1.0,1.0,1.0,1.0 +bm25,5,10,6.557,1.0,1.0,1.0,1.0 +bm25,13,10,9.12533,0.0,0.0,0.0,0.0 +bm25,36,10,8.81638,0.21841,0.5,0.16667,0.0 +bm25,42,10,13.62029,1.0,1.0,1.0,1.0 +bm25,48,10,12.43163,0.63093,1.0,0.5,1.0 +bm25,49,10,6.46029,1.0,1.0,1.0,1.0 +bm25,50,10,6.65588,1.0,1.0,1.0,1.0 +bm25,51,10,8.65617,1.0,1.0,1.0,1.0 +bm25,53,10,8.78,1.0,1.0,1.0,1.0 +bm25,54,10,11.20696,0.35621,1.0,0.16667,0.0 +bm25,56,10,12.52092,1.0,1.0,1.0,1.0 +bm25,57,10,12.28654,1.0,1.0,1.0,1.0 +bm25,70,10,5.50329,1.0,1.0,1.0,1.0 +bm25,72,10,7.31808,0.30103,1.0,0.11111,0.0 +bm25,75,10,11.90721,0.63093,1.0,0.5,1.0 +bm25,94,10,6.55983,1.0,1.0,1.0,1.0 +bm25,99,10,9.00704,1.0,1.0,1.0,1.0 +bm25,100,10,6.68696,1.0,1.0,1.0,1.0 +bm25,113,10,10.71208,1.0,1.0,1.0,1.0 +bm25,115,10,9.52062,1.0,1.0,1.0,1.0 +bm25,118,10,10.03517,1.0,1.0,1.0,1.0 +bm25,124,10,9.70771,1.0,1.0,1.0,1.0 +bm25,127,10,9.72154,0.5,1.0,0.33333,1.0 +bm25,128,10,6.696,0.5,1.0,0.33333,1.0 +bm25,129,10,12.25571,1.0,1.0,1.0,1.0 +bm25,130,10,12.36138,1.0,1.0,1.0,1.0 +bm25,132,10,5.56808,0.0,0.0,0.0,0.0 +bm25,133,10,16.35088,0.13121,0.2,0.2,0.0 +bm25,137,10,10.40683,1.0,1.0,1.0,1.0 +bm25,141,10,10.71213,1.0,1.0,1.0,1.0 +bm25,142,10,16.98696,1.0,1.0,1.0,1.0 +bm25,143,10,16.71137,1.0,1.0,1.0,1.0 +bm25,146,10,17.47621,1.0,1.0,1.0,1.0 +bm25,148,10,4.45529,1.0,1.0,1.0,1.0 +bm25,163,10,7.63083,1.0,1.0,1.0,1.0 +bm25,171,10,8.898,1.0,1.0,1.0,1.0 +bm25,179,10,7.91554,0.58557,0.5,1.0,1.0 +bm25,180,10,18.49875,1.0,1.0,1.0,1.0 +bm25,183,10,6.84083,0.31546,1.0,0.125,0.0 +bm25,185,10,7.81083,0.63093,1.0,0.5,1.0 +bm25,198,10,4.62796,0.5,1.0,0.33333,1.0 +bm25,208,10,6.52854,1.0,1.0,1.0,1.0 +bm25,212,10,6.32787,1.0,1.0,1.0,1.0 +bm25,213,10,10.65467,0.35621,1.0,0.16667,0.0 +bm25,216,10,8.19171,1.0,1.0,1.0,1.0 +bm25,217,10,8.41008,1.0,1.0,1.0,1.0 +bm25,218,10,6.95617,1.0,1.0,1.0,1.0 +bm25,219,10,8.09308,1.0,1.0,1.0,1.0 +bm25,230,10,12.46288,1.0,1.0,1.0,1.0 +bm25,232,10,10.43671,0.63093,1.0,0.5,1.0 +bm25,233,10,9.52825,1.0,1.0,1.0,1.0 +bm25,236,10,7.69738,1.0,1.0,1.0,1.0 +bm25,237,10,9.16971,1.0,1.0,1.0,1.0 +bm25,238,10,6.05642,0.0,0.0,0.0,0.0 +bm25,239,10,6.65754,0.0,0.0,0.0,0.0 +bm25,248,10,7.15842,1.0,1.0,1.0,1.0 +bm25,249,10,5.75867,0.63093,1.0,0.5,1.0 +bm25,261,10,9.44167,0.26407,0.5,0.25,0.0 +bm25,268,10,4.30621,0.63093,1.0,0.5,1.0 +bm25,269,10,4.63504,0.63093,1.0,0.5,1.0 +bm25,274,10,17.32125,1.0,1.0,1.0,1.0 +bm25,275,10,12.62708,0.46928,0.33333,1.0,1.0 +bm25,279,10,9.33954,1.0,1.0,1.0,1.0 +bm25,294,10,10.32971,1.0,1.0,1.0,1.0 +bm25,295,10,14.52179,1.0,1.0,1.0,1.0 +bm25,298,10,10.9165,1.0,1.0,1.0,1.0 +bm25,300,10,21.98475,0.43068,1.0,0.25,0.0 +bm25,303,10,12.02821,0.63093,1.0,0.5,1.0 +bm25,312,10,10.51087,0.38685,1.0,0.2,0.0 +bm25,314,10,18.47554,0.5,1.0,0.33333,1.0 +bm25,324,10,4.97979,0.43068,1.0,0.25,0.0 +bm25,327,10,9.15442,1.0,1.0,1.0,1.0 +bm25,338,10,5.30429,1.0,1.0,1.0,1.0 +bm25,343,10,14.67142,0.65092,1.0,0.5,1.0 +bm25,350,10,11.77887,1.0,1.0,1.0,1.0 +bm25,354,10,10.3255,1.0,1.0,1.0,1.0 +bm25,362,10,19.52054,1.0,1.0,1.0,1.0 +bm25,380,10,9.96708,1.0,1.0,1.0,1.0 +bm25,384,10,11.01463,0.0,0.0,0.0,0.0 +bm25,385,10,9.73304,0.30657,0.5,0.33333,1.0 +bm25,386,10,13.20963,1.0,1.0,1.0,1.0 +bm25,388,10,7.73675,0.5,1.0,0.33333,1.0 +bm25,399,10,9.24733,1.0,1.0,1.0,1.0 +bm25,410,10,8.22479,1.0,1.0,1.0,1.0 +bm25,411,10,8.33821,1.0,1.0,1.0,1.0 +bm25,415,10,10.76142,0.38685,1.0,0.2,0.0 +bm25,421,10,10.77092,0.0,0.0,0.0,0.0 +bm25,431,10,10.62113,0.0,0.0,0.0,0.0 +bm25,436,10,11.45054,1.0,1.0,1.0,1.0 +bm25,437,10,17.00304,0.0,0.0,0.0,0.0 +bm25,439,10,12.408,0.63093,1.0,0.5,1.0 +bm25,440,10,12.51183,0.63093,1.0,0.5,1.0 +bm25,443,10,8.66087,1.0,1.0,1.0,1.0 +bm25,452,10,7.93883,0.2372,0.5,0.2,0.0 +bm25,475,10,9.00746,0.0,0.0,0.0,0.0 +bm25,478,10,21.15417,0.63093,1.0,0.5,1.0 +bm25,491,10,10.974,1.0,1.0,1.0,1.0 +bm25,501,10,6.61212,0.63093,1.0,0.5,1.0 +bm25,502,10,12.79908,0.0,0.0,0.0,0.0 +bm25,507,10,13.74617,1.0,1.0,1.0,1.0 +bm25,508,10,9.52638,0.0,0.0,0.0,0.0 +bm25,513,10,5.74,1.0,1.0,1.0,1.0 +bm25,514,10,17.84558,0.0,0.0,0.0,0.0 +bm25,516,10,11.72475,1.0,1.0,1.0,1.0 +bm25,517,10,6.35446,0.0,0.0,0.0,0.0 +bm25,521,10,20.43704,1.0,1.0,1.0,1.0 +bm25,525,10,16.90304,1.0,1.0,1.0,1.0 +bm25,527,10,13.64296,0.63093,1.0,0.5,1.0 +bm25,528,10,21.96592,1.0,1.0,1.0,1.0 +bm25,532,10,5.72408,1.0,1.0,1.0,1.0 +bm25,533,10,4.47021,1.0,1.0,1.0,1.0 +bm25,535,10,7.9155,0.0,0.0,0.0,0.0 +bm25,536,10,6.91379,1.0,1.0,1.0,1.0 +bm25,539,10,5.32933,1.0,1.0,1.0,1.0 +bm25,540,10,7.64167,0.2372,0.5,0.2,0.0 +bm25,544,10,7.64479,0.0,0.0,0.0,0.0 +bm25,549,10,6.47546,1.0,1.0,1.0,1.0 +bm25,551,10,19.59433,0.43068,1.0,0.25,0.0 +bm25,552,10,17.62138,1.0,1.0,1.0,1.0 +bm25,554,10,10.76975,0.33333,1.0,0.14286,0.0 +bm25,560,10,12.2025,0.0,0.0,0.0,0.0 +bm25,569,10,8.45863,1.0,1.0,1.0,1.0 +bm25,575,10,9.42117,0.0,0.0,0.0,0.0 +bm25,577,10,17.29412,0.0,0.0,0.0,0.0 +bm25,578,10,9.9785,1.0,1.0,1.0,1.0 +bm25,587,10,22.13587,0.43068,1.0,0.25,0.0 +bm25,589,10,17.82683,1.0,1.0,1.0,1.0 +bm25,593,10,9.34433,1.0,1.0,1.0,1.0 +bm25,597,10,6.25708,0.93252,1.0,1.0,1.0 +bm25,598,10,16.17804,0.35621,1.0,0.16667,0.0 +bm25,613,10,9.06229,1.0,1.0,1.0,1.0 +bm25,619,10,12.54408,0.2372,0.5,0.2,0.0 +bm25,623,10,10.87537,0.0,0.0,0.0,0.0 +bm25,628,10,12.90367,1.0,1.0,1.0,1.0 +bm25,636,10,11.21292,0.5,1.0,0.33333,1.0 +bm25,637,10,11.26408,0.28906,1.0,0.1,0.0 +bm25,641,10,8.73496,0.61315,0.5,1.0,1.0 +bm25,644,10,6.50683,0.63093,1.0,0.5,1.0 +bm25,649,10,11.3445,1.0,1.0,1.0,1.0 +bm25,659,10,6.30754,1.0,1.0,1.0,1.0 +bm25,660,10,5.06671,0.38685,1.0,0.2,0.0 +bm25,674,10,9.68846,1.0,1.0,1.0,1.0 +bm25,684,10,9.99217,1.0,1.0,1.0,1.0 +bm25,690,10,16.99375,0.0,0.0,0.0,0.0 +bm25,691,10,12.09092,1.0,1.0,1.0,1.0 +bm25,692,10,9.06979,0.63093,1.0,0.5,1.0 +bm25,693,10,8.86821,1.0,1.0,1.0,1.0 +bm25,700,10,9.49017,0.63093,1.0,0.5,1.0 +bm25,702,10,9.24421,0.5,1.0,0.33333,1.0 +bm25,715,10,11.77279,0.0,0.0,0.0,0.0 +bm25,716,10,9.45425,0.0,0.0,0.0,0.0 +bm25,718,10,8.31563,0.63093,1.0,0.5,1.0 +bm25,721,10,11.119,1.0,1.0,1.0,1.0 +bm25,723,10,12.41312,1.0,1.0,1.0,1.0 +bm25,727,10,11.66942,0.63093,1.0,0.5,1.0 +bm25,728,10,8.87488,0.60526,1.0,0.5,1.0 +bm25,729,10,11.19046,0.63093,1.0,0.5,1.0 +bm25,742,10,6.77529,1.0,1.0,1.0,1.0 +bm25,743,10,3.66917,1.0,1.0,1.0,1.0 +bm25,744,10,14.23892,0.63093,1.0,0.5,1.0 +bm25,756,10,12.22271,0.30103,1.0,0.11111,0.0 +bm25,759,10,15.88192,1.0,1.0,1.0,1.0 +bm25,768,10,9.86246,0.0,0.0,0.0,0.0 +bm25,770,10,20.78954,1.0,1.0,1.0,1.0 +bm25,775,10,13.35675,0.0,0.0,0.0,0.0 +bm25,781,10,12.56117,0.63093,1.0,0.5,1.0 +bm25,783,10,13.21708,0.0,0.0,0.0,0.0 +bm25,784,10,13.03567,0.43068,1.0,0.25,0.0 +bm25,785,10,13.49808,0.63093,1.0,0.5,1.0 +bm25,793,10,5.51429,1.0,1.0,1.0,1.0 +bm25,800,10,14.40808,0.0,0.0,0.0,0.0 +bm25,805,10,6.42687,1.0,1.0,1.0,1.0 +bm25,808,10,8.18229,1.0,1.0,1.0,1.0 +bm25,811,10,12.11167,1.0,1.0,1.0,1.0 +bm25,814,10,19.66229,1.0,1.0,1.0,1.0 +bm25,820,10,6.64508,0.0,0.0,0.0,0.0 +bm25,821,10,6.24904,0.0,0.0,0.0,0.0 +bm25,823,10,6.282,1.0,1.0,1.0,1.0 +bm25,830,10,13.10858,0.0,0.0,0.0,0.0 +bm25,831,10,10.98267,0.0,0.0,0.0,0.0 +bm25,832,10,5.78733,1.0,1.0,1.0,1.0 +bm25,834,10,9.37025,0.0,0.0,0.0,0.0 +bm25,837,10,6.95521,1.0,1.0,1.0,1.0 +bm25,839,10,11.91804,1.0,1.0,1.0,1.0 +bm25,845,10,8.39592,1.0,1.0,1.0,1.0 +bm25,847,10,14.40804,1.0,1.0,1.0,1.0 +bm25,852,10,12.34604,1.0,1.0,1.0,1.0 +bm25,859,10,7.01308,0.63093,1.0,0.5,1.0 +bm25,870,10,3.46608,0.0,0.0,0.0,0.0 +bm25,873,10,6.50225,0.0,0.0,0.0,0.0 +bm25,879,10,10.19929,1.0,1.0,1.0,1.0 +bm25,880,10,7.96075,1.0,1.0,1.0,1.0 +bm25,882,10,9.14162,1.0,1.0,1.0,1.0 +bm25,887,10,10.21475,0.0,0.0,0.0,0.0 +bm25,903,10,8.58121,1.0,1.0,1.0,1.0 +bm25,904,10,17.90442,1.0,1.0,1.0,1.0 +bm25,907,10,14.32883,1.0,1.0,1.0,1.0 +bm25,911,10,12.61363,1.0,1.0,1.0,1.0 +bm25,913,10,6.34121,0.0,0.0,0.0,0.0 +bm25,914,10,6.32117,0.0,0.0,0.0,0.0 +bm25,921,10,8.55629,0.63093,1.0,0.5,1.0 +bm25,922,10,10.09417,1.0,1.0,1.0,1.0 +bm25,936,10,7.31738,1.0,1.0,1.0,1.0 +bm25,956,10,10.30046,0.5,1.0,0.33333,1.0 +bm25,957,10,9.55017,0.63093,1.0,0.5,1.0 +bm25,960,10,3.98088,1.0,1.0,1.0,1.0 +bm25,967,10,8.90921,0.91972,1.0,1.0,1.0 +bm25,971,10,18.38096,0.64446,1.0,0.33333,1.0 +bm25,975,10,8.75683,0.0,0.0,0.0,0.0 +bm25,982,10,16.84492,1.0,1.0,1.0,1.0 +bm25,985,10,11.26917,1.0,1.0,1.0,1.0 +bm25,993,10,7.20283,0.63093,1.0,0.5,1.0 +bm25,1012,10,8.09896,1.0,1.0,1.0,1.0 +bm25,1014,10,7.69629,1.0,1.0,1.0,1.0 +bm25,1019,10,7.26471,0.63093,1.0,0.5,1.0 +bm25,1020,10,20.24329,0.5,1.0,0.33333,1.0 +bm25,1021,10,22.52279,0.5,1.0,0.33333,1.0 +bm25,1024,10,8.74508,1.0,1.0,1.0,1.0 +bm25,1029,10,18.38763,0.76536,0.66667,1.0,1.0 +bm25,1041,10,14.01004,0.44131,1.0,0.25,0.0 +bm25,1049,10,9.60029,0.0,0.0,0.0,0.0 +bm25,1062,10,6.25758,1.0,1.0,1.0,1.0 +bm25,1086,10,14.55367,1.0,1.0,1.0,1.0 +bm25,1088,10,10.16742,0.0,0.0,0.0,0.0 +bm25,1089,10,12.32804,1.0,1.0,1.0,1.0 +bm25,1099,10,3.31221,0.5,1.0,0.33333,1.0 +bm25,1100,10,2.54975,0.63093,1.0,0.5,1.0 +bm25,1104,10,19.99312,1.0,1.0,1.0,1.0 +bm25,1107,10,8.63712,1.0,1.0,1.0,1.0 +bm25,1110,10,7.33592,0.0,0.0,0.0,0.0 +bm25,1121,10,10.49471,0.63093,1.0,0.5,1.0 +bm25,1130,10,14.23987,1.0,1.0,1.0,1.0 +bm25,1132,10,12.75217,0.2372,0.5,0.2,0.0 +bm25,1137,10,6.90671,1.0,1.0,1.0,1.0 +bm25,1140,10,9.08512,0.0,0.0,0.0,0.0 +bm25,1144,10,14.91333,1.0,1.0,1.0,1.0 +bm25,1146,10,9.16233,1.0,1.0,1.0,1.0 +bm25,1150,10,10.88888,1.0,1.0,1.0,1.0 +bm25,1163,10,8.54629,1.0,1.0,1.0,1.0 +bm25,1175,10,7.11708,0.0,0.0,0.0,0.0 +bm25,1179,10,8.70867,0.30103,1.0,0.11111,0.0 +bm25,1180,10,8.85662,0.43068,1.0,0.25,0.0 +bm25,1185,10,22.12721,1.0,1.0,1.0,1.0 +bm25,1187,10,19.30642,1.0,1.0,1.0,1.0 +bm25,1191,10,9.04775,0.0,0.0,0.0,0.0 +bm25,1194,10,17.59413,0.31546,1.0,0.125,0.0 +bm25,1196,10,10.17904,0.0,0.0,0.0,0.0 +bm25,1197,10,11.89888,0.0,0.0,0.0,0.0 +bm25,1199,10,15.53829,0.0,0.0,0.0,0.0 +bm25,1200,10,17.85446,0.0,0.0,0.0,0.0 +bm25,1202,10,11.73875,1.0,1.0,1.0,1.0 +bm25,1204,10,11.81083,1.0,1.0,1.0,1.0 +bm25,1207,10,16.33167,0.43068,1.0,0.25,0.0 +bm25,1213,10,11.20404,0.0,0.0,0.0,0.0 +bm25,1216,10,9.49037,1.0,1.0,1.0,1.0 +bm25,1221,10,13.64625,0.0,0.0,0.0,0.0 +bm25,1225,10,7.43471,1.0,1.0,1.0,1.0 +bm25,1226,10,13.09746,0.0,0.0,0.0,0.0 +bm25,1232,10,13.2855,1.0,1.0,1.0,1.0 +bm25,1241,10,8.08446,0.0,0.0,0.0,0.0 +bm25,1245,10,8.97358,0.31546,1.0,0.125,0.0 +bm25,1259,10,21.173,0.63093,1.0,0.5,1.0 +bm25,1262,10,11.89475,1.0,1.0,1.0,1.0 +bm25,1266,10,19.11404,0.5,1.0,0.33333,1.0 +bm25,1270,10,13.78192,1.0,1.0,1.0,1.0 +bm25,1271,10,19.20796,1.0,1.0,1.0,1.0 +bm25,1272,10,12.80029,0.5,1.0,0.33333,1.0 +bm25,1273,10,10.11775,1.0,1.0,1.0,1.0 +bm25,1274,10,18.59812,0.64478,1.0,0.5,1.0 +bm25,1278,10,13.415,0.0,0.0,0.0,0.0 +bm25,1279,10,12.14392,0.0,0.0,0.0,0.0 +bm25,1280,10,11.38646,0.0,0.0,0.0,0.0 +bm25,1281,10,9.47004,0.0,0.0,0.0,0.0 +bm25,1282,10,14.68167,1.0,1.0,1.0,1.0 +bm25,1290,10,9.53971,1.0,1.0,1.0,1.0 +bm25,1292,10,8.58329,1.0,1.0,1.0,1.0 +bm25,1298,10,17.92021,1.0,1.0,1.0,1.0 +bm25,1303,10,6.891,1.0,1.0,1.0,1.0 +bm25,1316,10,8.51675,0.0,0.0,0.0,0.0 +bm25,1319,10,8.72733,0.43068,1.0,0.25,0.0 +bm25,1320,10,12.37388,1.0,1.0,1.0,1.0 +bm25,1332,10,16.9415,0.0,0.0,0.0,0.0 +bm25,1335,10,7.17392,1.0,1.0,1.0,1.0 +bm25,1336,10,6.299,1.0,1.0,1.0,1.0 +bm25,1337,10,8.71008,1.0,1.0,1.0,1.0 +bm25,1339,10,11.96975,1.0,1.0,1.0,1.0 +bm25,1344,10,24.0675,0.63093,1.0,0.5,1.0 +bm25,1352,10,10.31625,1.0,1.0,1.0,1.0 +bm25,1359,10,16.3705,1.0,1.0,1.0,1.0 +bm25,1362,10,6.29817,0.5,1.0,0.33333,1.0 +bm25,1363,10,8.62188,1.0,1.0,1.0,1.0 +bm25,1368,10,7.95821,0.0,0.0,0.0,0.0 +bm25,1370,10,7.10079,0.38685,1.0,0.2,0.0 +bm25,1379,10,13.24675,0.33425,0.5,0.33333,1.0 +bm25,1382,10,6.72883,0.0,0.0,0.0,0.0 +bm25,1385,10,5.37075,1.0,1.0,1.0,1.0 +bm25,1389,10,6.86471,1.0,1.0,1.0,1.0 +bm25,1395,10,17.375,0.0,0.0,0.0,0.0 +dense+rerank,1,10,1164.74704,0.0,0.0,0.0,0.0 +dense+rerank,3,10,1149.89896,1.0,1.0,1.0,1.0 +dense+rerank,5,10,1210.23108,1.0,1.0,1.0,1.0 +dense+rerank,13,10,1173.55654,0.0,0.0,0.0,0.0 +dense+rerank,36,10,1204.18021,0.38896,1.0,0.14286,0.0 +dense+rerank,42,10,1189.6685,1.0,1.0,1.0,1.0 +dense+rerank,48,10,999.74204,0.0,0.0,0.0,0.0 +dense+rerank,49,10,1037.38692,1.0,1.0,1.0,1.0 +dense+rerank,50,10,1154.36312,1.0,1.0,1.0,1.0 +dense+rerank,51,10,1186.03575,1.0,1.0,1.0,1.0 +dense+rerank,53,10,1170.28496,1.0,1.0,1.0,1.0 +dense+rerank,54,10,1013.57137,0.35621,1.0,0.16667,0.0 +dense+rerank,56,10,1200.04358,1.0,1.0,1.0,1.0 +dense+rerank,57,10,1189.79617,1.0,1.0,1.0,1.0 +dense+rerank,70,10,1172.42912,0.0,0.0,0.0,0.0 +dense+rerank,72,10,1126.19392,1.0,1.0,1.0,1.0 +dense+rerank,75,10,1172.21296,1.0,1.0,1.0,1.0 +dense+rerank,94,10,1181.85613,0.31546,1.0,0.125,0.0 +dense+rerank,99,10,1260.17054,1.0,1.0,1.0,1.0 +dense+rerank,100,10,1126.2835,1.0,1.0,1.0,1.0 +dense+rerank,113,10,1229.3985,1.0,1.0,1.0,1.0 +dense+rerank,115,10,1171.14483,1.0,1.0,1.0,1.0 +dense+rerank,118,10,1283.48483,1.0,1.0,1.0,1.0 +dense+rerank,124,10,1199.00042,1.0,1.0,1.0,1.0 +dense+rerank,127,10,1232.48704,0.63093,1.0,0.5,1.0 +dense+rerank,128,10,1143.23225,0.0,0.0,0.0,0.0 +dense+rerank,129,10,1247.67867,1.0,1.0,1.0,1.0 +dense+rerank,130,10,1242.50129,1.0,1.0,1.0,1.0 +dense+rerank,132,10,1245.10967,0.0,0.0,0.0,0.0 +dense+rerank,133,10,1160.94,0.79725,0.8,1.0,1.0 +dense+rerank,137,10,1193.79567,1.0,1.0,1.0,1.0 +dense+rerank,141,10,1190.54425,0.91972,1.0,1.0,1.0 +dense+rerank,142,10,1288.3135,1.0,1.0,1.0,1.0 +dense+rerank,143,10,1209.04662,1.0,1.0,1.0,1.0 +dense+rerank,146,10,1292.74683,1.0,1.0,1.0,1.0 +dense+rerank,148,10,1304.69842,1.0,1.0,1.0,1.0 +dense+rerank,163,10,1262.45938,0.63093,1.0,0.5,1.0 +dense+rerank,171,10,1303.12475,1.0,1.0,1.0,1.0 +dense+rerank,179,10,1251.769,0.90522,1.0,1.0,1.0 +dense+rerank,180,10,1248.87904,1.0,1.0,1.0,1.0 +dense+rerank,183,10,1146.40262,1.0,1.0,1.0,1.0 +dense+rerank,185,10,1244.83146,0.43068,1.0,0.25,0.0 +dense+rerank,198,10,1263.62062,0.0,0.0,0.0,0.0 +dense+rerank,208,10,1270.44004,1.0,1.0,1.0,1.0 +dense+rerank,212,10,1242.01879,0.5,1.0,0.33333,1.0 +dense+rerank,213,10,1193.83383,0.0,0.0,0.0,0.0 +dense+rerank,216,10,812.28421,1.0,1.0,1.0,1.0 +dense+rerank,217,10,1280.79958,1.0,1.0,1.0,1.0 +dense+rerank,218,10,1202.15858,1.0,1.0,1.0,1.0 +dense+rerank,219,10,1360.35721,1.0,1.0,1.0,1.0 +dense+rerank,230,10,1291.68358,1.0,1.0,1.0,1.0 +dense+rerank,232,10,1273.29338,0.63093,1.0,0.5,1.0 +dense+rerank,233,10,1274.32258,1.0,1.0,1.0,1.0 +dense+rerank,236,10,1268.18083,1.0,1.0,1.0,1.0 +dense+rerank,237,10,1236.86712,0.43068,1.0,0.25,0.0 +dense+rerank,238,10,1214.64275,0.0,0.0,0.0,0.0 +dense+rerank,239,10,1284.28708,0.0,0.0,0.0,0.0 +dense+rerank,248,10,1207.87196,1.0,1.0,1.0,1.0 +dense+rerank,249,10,1175.85217,1.0,1.0,1.0,1.0 +dense+rerank,261,10,1345.66133,0.38685,0.5,0.5,1.0 +dense+rerank,268,10,1288.26142,0.63093,1.0,0.5,1.0 +dense+rerank,269,10,1322.02779,0.63093,1.0,0.5,1.0 +dense+rerank,274,10,1282.74896,1.0,1.0,1.0,1.0 +dense+rerank,275,10,1294.5355,0.87108,1.0,1.0,1.0 +dense+rerank,279,10,1273.56879,1.0,1.0,1.0,1.0 +dense+rerank,294,10,798.63746,1.0,1.0,1.0,1.0 +dense+rerank,295,10,1456.23821,1.0,1.0,1.0,1.0 +dense+rerank,298,10,1363.05433,1.0,1.0,1.0,1.0 +dense+rerank,300,10,817.32817,0.5,1.0,0.33333,1.0 +dense+rerank,303,10,1449.93475,0.0,0.0,0.0,0.0 +dense+rerank,312,10,1179.88392,0.0,0.0,0.0,0.0 +dense+rerank,314,10,1453.115,0.0,0.0,0.0,0.0 +dense+rerank,324,10,1409.98917,0.38685,1.0,0.2,0.0 +dense+rerank,327,10,1045.72721,1.0,1.0,1.0,1.0 +dense+rerank,338,10,1389.70204,1.0,1.0,1.0,1.0 +dense+rerank,343,10,1010.37233,0.57064,1.0,0.33333,1.0 +dense+rerank,350,10,1328.97996,1.0,1.0,1.0,1.0 +dense+rerank,354,10,1362.54258,1.0,1.0,1.0,1.0 +dense+rerank,362,10,1328.57362,1.0,1.0,1.0,1.0 +dense+rerank,380,10,1362.93321,1.0,1.0,1.0,1.0 +dense+rerank,384,10,1265.12367,0.0,0.0,0.0,0.0 +dense+rerank,385,10,1251.88733,0.69343,1.0,0.5,1.0 +dense+rerank,386,10,1378.97354,1.0,1.0,1.0,1.0 +dense+rerank,388,10,1054.36792,0.5,1.0,0.33333,1.0 +dense+rerank,399,10,1389.52846,1.0,1.0,1.0,1.0 +dense+rerank,410,10,1318.78333,1.0,1.0,1.0,1.0 +dense+rerank,411,10,1436.97067,1.0,1.0,1.0,1.0 +dense+rerank,415,10,1358.66483,0.5,1.0,0.33333,1.0 +dense+rerank,421,10,1358.73267,0.0,0.0,0.0,0.0 +dense+rerank,431,10,1414.62425,0.30103,1.0,0.11111,0.0 +dense+rerank,436,10,651.75154,1.0,1.0,1.0,1.0 +dense+rerank,437,10,1266.61108,0.0,0.0,0.0,0.0 +dense+rerank,439,10,1289.42471,1.0,1.0,1.0,1.0 +dense+rerank,440,10,1366.70037,1.0,1.0,1.0,1.0 +dense+rerank,443,10,1408.50446,1.0,1.0,1.0,1.0 +dense+rerank,452,10,1369.38654,0.2372,0.5,0.2,0.0 +dense+rerank,475,10,1364.42058,0.38685,1.0,0.2,0.0 +dense+rerank,478,10,1278.30271,1.0,1.0,1.0,1.0 +dense+rerank,491,10,1442.156,1.0,1.0,1.0,1.0 +dense+rerank,501,10,1349.00996,1.0,1.0,1.0,1.0 +dense+rerank,502,10,1298.45492,0.0,0.0,0.0,0.0 +dense+rerank,507,10,1276.08721,0.63093,1.0,0.5,1.0 +dense+rerank,508,10,1407.56175,0.0,0.0,0.0,0.0 +dense+rerank,513,10,1349.00808,0.63093,1.0,0.5,1.0 +dense+rerank,514,10,1330.037,0.5,1.0,0.33333,1.0 +dense+rerank,516,10,1404.32242,1.0,1.0,1.0,1.0 +dense+rerank,517,10,1229.73829,0.0,0.0,0.0,0.0 +dense+rerank,521,10,1342.79871,1.0,1.0,1.0,1.0 +dense+rerank,525,10,1308.66717,1.0,1.0,1.0,1.0 +dense+rerank,527,10,1324.32996,0.35621,1.0,0.16667,0.0 +dense+rerank,528,10,1375.58004,0.5,1.0,0.33333,1.0 +dense+rerank,532,10,1350.72183,1.0,1.0,1.0,1.0 +dense+rerank,533,10,1384.47737,1.0,1.0,1.0,1.0 +dense+rerank,535,10,1345.391,0.0,0.0,0.0,0.0 +dense+rerank,536,10,1319.34346,1.0,1.0,1.0,1.0 +dense+rerank,539,10,1307.54392,1.0,1.0,1.0,1.0 +dense+rerank,540,10,1261.76425,0.30657,0.5,0.33333,1.0 +dense+rerank,544,10,1229.79079,0.38685,1.0,0.2,0.0 +dense+rerank,549,10,1369.06629,1.0,1.0,1.0,1.0 +dense+rerank,551,10,1262.52029,0.5,1.0,0.33333,1.0 +dense+rerank,552,10,1443.57017,1.0,1.0,1.0,1.0 +dense+rerank,554,10,1327.63329,0.35621,1.0,0.16667,0.0 +dense+rerank,560,10,1312.74204,0.0,0.0,0.0,0.0 +dense+rerank,569,10,1362.26946,1.0,1.0,1.0,1.0 +dense+rerank,575,10,1165.10571,1.0,1.0,1.0,1.0 +dense+rerank,577,10,1273.71154,0.0,0.0,0.0,0.0 +dense+rerank,578,10,1469.10667,1.0,1.0,1.0,1.0 +dense+rerank,587,10,1346.09646,0.0,0.0,0.0,0.0 +dense+rerank,589,10,1244.33604,1.0,1.0,1.0,1.0 +dense+rerank,593,10,1238.33179,1.0,1.0,1.0,1.0 +dense+rerank,597,10,1281.07267,0.88546,1.0,1.0,1.0 +dense+rerank,598,10,1330.53925,0.33333,1.0,0.14286,0.0 +dense+rerank,613,10,1385.325,1.0,1.0,1.0,1.0 +dense+rerank,619,10,1280.74108,0.61315,0.5,1.0,1.0 +dense+rerank,623,10,1338.14771,0.0,0.0,0.0,0.0 +dense+rerank,628,10,1338.001,1.0,1.0,1.0,1.0 +dense+rerank,636,10,1499.57896,1.0,1.0,1.0,1.0 +dense+rerank,637,10,1493.7895,0.63093,1.0,0.5,1.0 +dense+rerank,641,10,1520.55346,0.79772,1.0,1.0,1.0 +dense+rerank,644,10,1420.61979,1.0,1.0,1.0,1.0 +dense+rerank,649,10,1398.37696,1.0,1.0,1.0,1.0 +dense+rerank,659,10,1399.56971,0.35621,1.0,0.16667,0.0 +dense+rerank,660,10,1447.65629,0.0,0.0,0.0,0.0 +dense+rerank,674,10,1531.54163,1.0,1.0,1.0,1.0 +dense+rerank,684,10,1558.39863,0.43068,1.0,0.25,0.0 +dense+rerank,690,10,1400.99817,0.0,0.0,0.0,0.0 +dense+rerank,691,10,1428.70608,0.63093,1.0,0.5,1.0 +dense+rerank,692,10,1605.89088,1.0,1.0,1.0,1.0 +dense+rerank,693,10,1427.73683,1.0,1.0,1.0,1.0 +dense+rerank,700,10,1353.06533,0.33333,1.0,0.14286,0.0 +dense+rerank,702,10,1436.66742,0.31546,1.0,0.125,0.0 +dense+rerank,715,10,1512.52646,0.0,0.0,0.0,0.0 +dense+rerank,716,10,1567.04563,0.0,0.0,0.0,0.0 +dense+rerank,718,10,1620.40863,1.0,1.0,1.0,1.0 +dense+rerank,721,10,1462.11842,1.0,1.0,1.0,1.0 +dense+rerank,723,10,1621.84254,1.0,1.0,1.0,1.0 +dense+rerank,727,10,1680.79679,0.33333,1.0,0.14286,0.0 +dense+rerank,728,10,1544.09521,0.81753,1.0,1.0,1.0 +dense+rerank,729,10,1458.93746,1.0,1.0,1.0,1.0 +dense+rerank,742,10,1526.3035,1.0,1.0,1.0,1.0 +dense+rerank,743,10,1504.42971,1.0,1.0,1.0,1.0 +dense+rerank,744,10,1447.91087,0.63093,1.0,0.5,1.0 +dense+rerank,756,10,782.60717,0.43068,1.0,0.25,0.0 +dense+rerank,759,10,1662.98054,1.0,1.0,1.0,1.0 +dense+rerank,768,10,1421.75942,0.0,0.0,0.0,0.0 +dense+rerank,770,10,1596.69371,1.0,1.0,1.0,1.0 +dense+rerank,775,10,1389.14492,0.33333,1.0,0.14286,0.0 +dense+rerank,781,10,1384.54154,0.63093,1.0,0.5,1.0 +dense+rerank,783,10,1395.91663,0.31546,1.0,0.125,0.0 +dense+rerank,784,10,1655.784,1.0,1.0,1.0,1.0 +dense+rerank,785,10,1036.99725,0.0,0.0,0.0,0.0 +dense+rerank,793,10,1730.41329,1.0,1.0,1.0,1.0 +dense+rerank,800,10,817.81583,0.63093,1.0,0.5,1.0 +dense+rerank,805,10,1552.09312,1.0,1.0,1.0,1.0 +dense+rerank,808,10,1628.57375,1.0,1.0,1.0,1.0 +dense+rerank,811,10,1427.53746,1.0,1.0,1.0,1.0 +dense+rerank,814,10,1444.62083,1.0,1.0,1.0,1.0 +dense+rerank,820,10,714.65846,0.0,0.0,0.0,0.0 +dense+rerank,821,10,710.973,0.0,0.0,0.0,0.0 +dense+rerank,823,10,1298.86967,1.0,1.0,1.0,1.0 +dense+rerank,830,10,1026.77167,0.0,0.0,0.0,0.0 +dense+rerank,831,10,1486.5275,0.0,0.0,0.0,0.0 +dense+rerank,832,10,1558.72138,1.0,1.0,1.0,1.0 +dense+rerank,834,10,1437.42304,0.0,0.0,0.0,0.0 +dense+rerank,837,10,1559.43825,0.63093,1.0,0.5,1.0 +dense+rerank,839,10,1312.49071,1.0,1.0,1.0,1.0 +dense+rerank,845,10,1502.46054,1.0,1.0,1.0,1.0 +dense+rerank,847,10,1536.27963,1.0,1.0,1.0,1.0 +dense+rerank,852,10,1594.31162,0.63093,1.0,0.5,1.0 +dense+rerank,859,10,1614.851,0.63093,1.0,0.5,1.0 +dense+rerank,870,10,1691.98,0.0,0.0,0.0,0.0 +dense+rerank,873,10,1693.33629,0.41369,0.6,0.33333,1.0 +dense+rerank,879,10,1550.26721,1.0,1.0,1.0,1.0 +dense+rerank,880,10,1390.90179,1.0,1.0,1.0,1.0 +dense+rerank,882,10,1553.79046,1.0,1.0,1.0,1.0 +dense+rerank,887,10,1522.0315,0.0,0.0,0.0,0.0 +dense+rerank,903,10,1516.88063,1.0,1.0,1.0,1.0 +dense+rerank,904,10,1426.56246,1.0,1.0,1.0,1.0 +dense+rerank,907,10,1649.30354,1.0,1.0,1.0,1.0 +dense+rerank,911,10,1915.88021,1.0,1.0,1.0,1.0 +dense+rerank,913,10,1503.58329,0.0,0.0,0.0,0.0 +dense+rerank,914,10,1507.30892,0.0,0.0,0.0,0.0 +dense+rerank,921,10,1116.68029,1.0,1.0,1.0,1.0 +dense+rerank,922,10,1043.52371,1.0,1.0,1.0,1.0 +dense+rerank,936,10,1069.47833,1.0,1.0,1.0,1.0 +dense+rerank,956,10,1042.38504,1.0,1.0,1.0,1.0 +dense+rerank,957,10,789.03937,1.0,1.0,1.0,1.0 +dense+rerank,960,10,1058.13787,1.0,1.0,1.0,1.0 +dense+rerank,967,10,1079.37217,0.91972,1.0,1.0,1.0 +dense+rerank,971,10,1076.67017,0.95583,1.0,1.0,1.0 +dense+rerank,975,10,1148.21683,0.0,0.0,0.0,0.0 +dense+rerank,982,10,959.22658,1.0,1.0,1.0,1.0 +dense+rerank,985,10,894.26108,1.0,1.0,1.0,1.0 +dense+rerank,993,10,1202.96292,1.0,1.0,1.0,1.0 +dense+rerank,1012,10,1150.18937,1.0,1.0,1.0,1.0 +dense+rerank,1014,10,1194.33454,1.0,1.0,1.0,1.0 +dense+rerank,1019,10,1130.87971,0.63093,1.0,0.5,1.0 +dense+rerank,1020,10,1167.80462,0.63093,1.0,0.5,1.0 +dense+rerank,1021,10,1151.98229,0.63093,1.0,0.5,1.0 +dense+rerank,1024,10,1129.57496,0.0,0.0,0.0,0.0 +dense+rerank,1029,10,1171.71537,0.76536,0.66667,1.0,1.0 +dense+rerank,1041,10,1184.10654,0.21841,0.5,0.16667,0.0 +dense+rerank,1049,10,1199.65871,0.43068,1.0,0.25,0.0 +dense+rerank,1062,10,1174.41646,1.0,1.0,1.0,1.0 +dense+rerank,1086,10,1332.60308,1.0,1.0,1.0,1.0 +dense+rerank,1088,10,1181.78238,1.0,1.0,1.0,1.0 +dense+rerank,1089,10,931.09142,1.0,1.0,1.0,1.0 +dense+rerank,1099,10,1123.3925,0.43068,1.0,0.25,0.0 +dense+rerank,1100,10,1439.32667,0.43068,1.0,0.25,0.0 +dense+rerank,1104,10,1224.28546,1.0,1.0,1.0,1.0 +dense+rerank,1107,10,1168.41004,1.0,1.0,1.0,1.0 +dense+rerank,1110,10,1113.00596,0.0,0.0,0.0,0.0 +dense+rerank,1121,10,1062.03075,1.0,1.0,1.0,1.0 +dense+rerank,1130,10,1138.63721,1.0,1.0,1.0,1.0 +dense+rerank,1132,10,1184.21871,0.61315,0.5,1.0,1.0 +dense+rerank,1137,10,1175.11967,1.0,1.0,1.0,1.0 +dense+rerank,1140,10,1185.94208,1.0,1.0,1.0,1.0 +dense+rerank,1144,10,1188.16333,1.0,1.0,1.0,1.0 +dense+rerank,1146,10,1179.70188,1.0,1.0,1.0,1.0 +dense+rerank,1150,10,1348.93183,1.0,1.0,1.0,1.0 +dense+rerank,1163,10,1279.18933,1.0,1.0,1.0,1.0 +dense+rerank,1175,10,1199.85358,0.0,0.0,0.0,0.0 +dense+rerank,1179,10,1171.50608,0.0,0.0,0.0,0.0 +dense+rerank,1180,10,1149.04679,1.0,1.0,1.0,1.0 +dense+rerank,1185,10,1160.12742,1.0,1.0,1.0,1.0 +dense+rerank,1187,10,874.23287,1.0,1.0,1.0,1.0 +dense+rerank,1191,10,822.64475,0.38685,1.0,0.2,0.0 +dense+rerank,1194,10,809.30554,1.0,1.0,1.0,1.0 +dense+rerank,1196,10,1192.98117,0.43068,1.0,0.25,0.0 +dense+rerank,1197,10,1222.77792,0.43068,1.0,0.25,0.0 +dense+rerank,1199,10,1196.42829,0.0,0.0,0.0,0.0 +dense+rerank,1200,10,1188.92908,0.0,0.0,0.0,0.0 +dense+rerank,1202,10,1181.927,0.63093,1.0,0.5,1.0 +dense+rerank,1204,10,1244.24571,0.43068,1.0,0.25,0.0 +dense+rerank,1207,10,1173.73871,1.0,1.0,1.0,1.0 +dense+rerank,1213,10,1213.05729,0.0,0.0,0.0,0.0 +dense+rerank,1216,10,1160.23513,1.0,1.0,1.0,1.0 +dense+rerank,1221,10,1144.6505,0.38685,1.0,0.2,0.0 +dense+rerank,1225,10,1235.39521,1.0,1.0,1.0,1.0 +dense+rerank,1226,10,1259.3375,0.0,0.0,0.0,0.0 +dense+rerank,1232,10,1231.72146,1.0,1.0,1.0,1.0 +dense+rerank,1241,10,1211.35721,1.0,1.0,1.0,1.0 +dense+rerank,1245,10,1224.15463,0.0,0.0,0.0,0.0 +dense+rerank,1259,10,1030.63062,1.0,1.0,1.0,1.0 +dense+rerank,1262,10,1195.19571,1.0,1.0,1.0,1.0 +dense+rerank,1266,10,1132.64479,0.5,1.0,0.33333,1.0 +dense+rerank,1270,10,1169.06608,1.0,1.0,1.0,1.0 +dense+rerank,1271,10,1038.27225,1.0,1.0,1.0,1.0 +dense+rerank,1272,10,1038.39196,0.5,1.0,0.33333,1.0 +dense+rerank,1273,10,1282.12179,1.0,1.0,1.0,1.0 +dense+rerank,1274,10,1206.79929,0.93252,1.0,1.0,1.0 +dense+rerank,1278,10,1202.67479,0.31546,1.0,0.125,0.0 +dense+rerank,1279,10,1165.00675,0.35621,1.0,0.16667,0.0 +dense+rerank,1280,10,1185.44825,0.35621,1.0,0.16667,0.0 +dense+rerank,1281,10,1207.36079,0.31546,1.0,0.125,0.0 +dense+rerank,1282,10,1210.55125,1.0,1.0,1.0,1.0 +dense+rerank,1290,10,1114.97221,1.0,1.0,1.0,1.0 +dense+rerank,1292,10,1191.17238,0.5,1.0,0.33333,1.0 +dense+rerank,1298,10,1195.6185,1.0,1.0,1.0,1.0 +dense+rerank,1303,10,1176.33179,1.0,1.0,1.0,1.0 +dense+rerank,1316,10,1206.21308,0.0,0.0,0.0,0.0 +dense+rerank,1319,10,1140.62217,1.0,1.0,1.0,1.0 +dense+rerank,1320,10,1197.28671,1.0,1.0,1.0,1.0 +dense+rerank,1332,10,1131.50117,0.0,0.0,0.0,0.0 +dense+rerank,1335,10,1236.75325,1.0,1.0,1.0,1.0 +dense+rerank,1336,10,1128.89583,1.0,1.0,1.0,1.0 +dense+rerank,1337,10,1190.35675,0.63093,1.0,0.5,1.0 +dense+rerank,1339,10,1189.54912,1.0,1.0,1.0,1.0 +dense+rerank,1344,10,1174.77358,0.5,1.0,0.33333,1.0 +dense+rerank,1352,10,1136.08954,1.0,1.0,1.0,1.0 +dense+rerank,1359,10,1201.49696,1.0,1.0,1.0,1.0 +dense+rerank,1362,10,1155.16692,0.0,0.0,0.0,0.0 +dense+rerank,1363,10,1169.0015,0.0,0.0,0.0,0.0 +dense+rerank,1368,10,1171.75837,1.0,1.0,1.0,1.0 +dense+rerank,1370,10,1228.99517,0.63093,1.0,0.5,1.0 +dense+rerank,1379,10,1245.41196,0.83268,1.0,1.0,1.0 +dense+rerank,1382,10,1141.86508,1.0,1.0,1.0,1.0 +dense+rerank,1385,10,1246.95488,1.0,1.0,1.0,1.0 +dense+rerank,1389,10,1184.89296,1.0,1.0,1.0,1.0 +dense+rerank,1395,10,1176.43087,1.0,1.0,1.0,1.0 +hybrid+rerank,1,10,1732.08971,0.0,0.0,0.0,0.0 +hybrid+rerank,3,10,1646.98683,1.0,1.0,1.0,1.0 +hybrid+rerank,5,10,1876.24242,1.0,1.0,1.0,1.0 +hybrid+rerank,13,10,1709.75288,0.0,0.0,0.0,0.0 +hybrid+rerank,36,10,1536.35825,0.38896,1.0,0.14286,0.0 +hybrid+rerank,42,10,1803.20808,1.0,1.0,1.0,1.0 +hybrid+rerank,48,10,2289.47888,0.5,1.0,0.33333,1.0 +hybrid+rerank,49,10,1735.75792,1.0,1.0,1.0,1.0 +hybrid+rerank,50,10,1461.63221,1.0,1.0,1.0,1.0 +hybrid+rerank,51,10,1714.94,0.63093,1.0,0.5,1.0 +hybrid+rerank,53,10,1792.89125,1.0,1.0,1.0,1.0 +hybrid+rerank,54,10,1606.66675,0.30103,1.0,0.11111,0.0 +hybrid+rerank,56,10,1868.75763,1.0,1.0,1.0,1.0 +hybrid+rerank,57,10,1953.92233,1.0,1.0,1.0,1.0 +hybrid+rerank,70,10,1556.40196,0.0,0.0,0.0,0.0 +hybrid+rerank,72,10,2005.05975,1.0,1.0,1.0,1.0 +hybrid+rerank,75,10,1909.08375,1.0,1.0,1.0,1.0 +hybrid+rerank,94,10,2052.03525,0.0,0.0,0.0,0.0 +hybrid+rerank,99,10,1770.28617,1.0,1.0,1.0,1.0 +hybrid+rerank,100,10,1557.19321,1.0,1.0,1.0,1.0 +hybrid+rerank,113,10,1873.48704,1.0,1.0,1.0,1.0 +hybrid+rerank,115,10,1965.49137,1.0,1.0,1.0,1.0 +hybrid+rerank,118,10,1846.39221,1.0,1.0,1.0,1.0 +hybrid+rerank,124,10,1539.2215,1.0,1.0,1.0,1.0 +hybrid+rerank,127,10,2093.15987,0.63093,1.0,0.5,1.0 +hybrid+rerank,128,10,2020.56325,0.31546,1.0,0.125,0.0 +hybrid+rerank,129,10,2171.84229,1.0,1.0,1.0,1.0 +hybrid+rerank,130,10,1999.11925,1.0,1.0,1.0,1.0 +hybrid+rerank,132,10,2021.86267,0.0,0.0,0.0,0.0 +hybrid+rerank,133,10,2010.36046,0.51477,0.6,0.5,1.0 +hybrid+rerank,137,10,2092.97171,1.0,1.0,1.0,1.0 +hybrid+rerank,141,10,1856.62742,0.91972,1.0,1.0,1.0 +hybrid+rerank,142,10,1875.78092,1.0,1.0,1.0,1.0 +hybrid+rerank,143,10,1527.341,1.0,1.0,1.0,1.0 +hybrid+rerank,146,10,1575.78196,1.0,1.0,1.0,1.0 +hybrid+rerank,148,10,1673.90925,1.0,1.0,1.0,1.0 +hybrid+rerank,163,10,1581.00871,0.63093,1.0,0.5,1.0 +hybrid+rerank,171,10,1996.78358,1.0,1.0,1.0,1.0 +hybrid+rerank,179,10,2085.88908,0.90522,1.0,1.0,1.0 +hybrid+rerank,180,10,2247.75908,1.0,1.0,1.0,1.0 +hybrid+rerank,183,10,1620.61329,1.0,1.0,1.0,1.0 +hybrid+rerank,185,10,2344.46346,0.43068,1.0,0.25,0.0 +hybrid+rerank,198,10,2492.69279,0.28906,1.0,0.1,0.0 +hybrid+rerank,208,10,2277.94579,1.0,1.0,1.0,1.0 +hybrid+rerank,212,10,2163.79192,0.5,1.0,0.33333,1.0 +hybrid+rerank,213,10,2468.85108,0.0,0.0,0.0,0.0 +hybrid+rerank,216,10,2374.58796,1.0,1.0,1.0,1.0 +hybrid+rerank,217,10,2099.65983,1.0,1.0,1.0,1.0 +hybrid+rerank,218,10,1845.28342,1.0,1.0,1.0,1.0 +hybrid+rerank,219,10,2097.94825,1.0,1.0,1.0,1.0 +hybrid+rerank,230,10,2032.91379,1.0,1.0,1.0,1.0 +hybrid+rerank,232,10,1977.6715,0.63093,1.0,0.5,1.0 +hybrid+rerank,233,10,2366.12529,1.0,1.0,1.0,1.0 +hybrid+rerank,236,10,2314.41087,1.0,1.0,1.0,1.0 +hybrid+rerank,237,10,2320.13712,0.43068,1.0,0.25,0.0 +hybrid+rerank,238,10,1965.80146,0.0,0.0,0.0,0.0 +hybrid+rerank,239,10,2139.92233,0.0,0.0,0.0,0.0 +hybrid+rerank,248,10,2207.22204,1.0,1.0,1.0,1.0 +hybrid+rerank,249,10,2207.02312,1.0,1.0,1.0,1.0 +hybrid+rerank,261,10,2154.84883,0.38685,0.5,0.5,1.0 +hybrid+rerank,268,10,2037.27063,0.63093,1.0,0.5,1.0 +hybrid+rerank,269,10,1949.67112,0.63093,1.0,0.5,1.0 +hybrid+rerank,274,10,2767.29788,1.0,1.0,1.0,1.0 +hybrid+rerank,275,10,1997.08329,0.87108,1.0,1.0,1.0 +hybrid+rerank,279,10,1874.77021,1.0,1.0,1.0,1.0 +hybrid+rerank,294,10,1810.799,1.0,1.0,1.0,1.0 +hybrid+rerank,295,10,1943.82571,1.0,1.0,1.0,1.0 +hybrid+rerank,298,10,1894.18025,1.0,1.0,1.0,1.0 +hybrid+rerank,300,10,2116.9325,0.5,1.0,0.33333,1.0 +hybrid+rerank,303,10,2123.45808,0.0,0.0,0.0,0.0 +hybrid+rerank,312,10,2221.33758,0.0,0.0,0.0,0.0 +hybrid+rerank,314,10,2056.68183,0.0,0.0,0.0,0.0 +hybrid+rerank,324,10,2095.70558,0.0,0.0,0.0,0.0 +hybrid+rerank,327,10,2435.54046,1.0,1.0,1.0,1.0 +hybrid+rerank,338,10,2999.51158,1.0,1.0,1.0,1.0 +hybrid+rerank,343,10,2909.25675,0.57064,1.0,0.33333,1.0 +hybrid+rerank,350,10,2198.02688,1.0,1.0,1.0,1.0 +hybrid+rerank,354,10,2006.88325,1.0,1.0,1.0,1.0 +hybrid+rerank,362,10,2526.6015,1.0,1.0,1.0,1.0 +hybrid+rerank,380,10,2189.49396,1.0,1.0,1.0,1.0 +hybrid+rerank,384,10,2284.35429,0.31546,1.0,0.125,0.0 +hybrid+rerank,385,10,1996.28179,0.65092,1.0,0.5,1.0 +hybrid+rerank,386,10,2425.08496,1.0,1.0,1.0,1.0 +hybrid+rerank,388,10,2238.445,0.30103,1.0,0.11111,0.0 +hybrid+rerank,399,10,1745.08275,1.0,1.0,1.0,1.0 +hybrid+rerank,410,10,2361.19025,1.0,1.0,1.0,1.0 +hybrid+rerank,411,10,2153.55079,1.0,1.0,1.0,1.0 +hybrid+rerank,415,10,2261.52154,0.5,1.0,0.33333,1.0 +hybrid+rerank,421,10,2250.96175,0.0,0.0,0.0,0.0 +hybrid+rerank,431,10,2276.14775,0.0,0.0,0.0,0.0 +hybrid+rerank,436,10,2142.26146,1.0,1.0,1.0,1.0 +hybrid+rerank,437,10,2257.69933,0.0,0.0,0.0,0.0 +hybrid+rerank,439,10,2230.15496,1.0,1.0,1.0,1.0 +hybrid+rerank,440,10,2259.89433,1.0,1.0,1.0,1.0 +hybrid+rerank,443,10,1807.78396,1.0,1.0,1.0,1.0 +hybrid+rerank,452,10,2285.21912,0.60526,1.0,0.5,1.0 +hybrid+rerank,475,10,2211.55258,0.38685,1.0,0.2,0.0 +hybrid+rerank,478,10,2094.68071,1.0,1.0,1.0,1.0 +hybrid+rerank,491,10,2235.88425,1.0,1.0,1.0,1.0 +hybrid+rerank,501,10,1531.05904,1.0,1.0,1.0,1.0 +hybrid+rerank,502,10,2162.19608,0.0,0.0,0.0,0.0 +hybrid+rerank,507,10,2247.03458,0.63093,1.0,0.5,1.0 +hybrid+rerank,508,10,1761.12492,0.0,0.0,0.0,0.0 +hybrid+rerank,513,10,2353.59158,0.63093,1.0,0.5,1.0 +hybrid+rerank,514,10,2183.48696,0.5,1.0,0.33333,1.0 +hybrid+rerank,516,10,1768.87467,1.0,1.0,1.0,1.0 +hybrid+rerank,517,10,2638.72475,0.35621,1.0,0.16667,0.0 +hybrid+rerank,521,10,2711.97692,1.0,1.0,1.0,1.0 +hybrid+rerank,525,10,2323.19229,1.0,1.0,1.0,1.0 +hybrid+rerank,527,10,1540.73217,0.28906,1.0,0.1,0.0 +hybrid+rerank,528,10,2340.531,0.5,1.0,0.33333,1.0 +hybrid+rerank,532,10,2879.06729,1.0,1.0,1.0,1.0 +hybrid+rerank,533,10,2458.21729,1.0,1.0,1.0,1.0 +hybrid+rerank,535,10,2696.85025,0.0,0.0,0.0,0.0 +hybrid+rerank,536,10,2471.83121,1.0,1.0,1.0,1.0 +hybrid+rerank,539,10,2099.48321,1.0,1.0,1.0,1.0 +hybrid+rerank,540,10,2245.344,0.30657,0.5,0.33333,1.0 +hybrid+rerank,544,10,2252.39008,0.38685,1.0,0.2,0.0 +hybrid+rerank,549,10,2006.81679,1.0,1.0,1.0,1.0 +hybrid+rerank,551,10,2116.95979,0.5,1.0,0.33333,1.0 +hybrid+rerank,552,10,2055.93888,1.0,1.0,1.0,1.0 +hybrid+rerank,554,10,2224.99846,0.0,0.0,0.0,0.0 +hybrid+rerank,560,10,2076.53108,0.0,0.0,0.0,0.0 +hybrid+rerank,569,10,2116.65838,1.0,1.0,1.0,1.0 +hybrid+rerank,575,10,2104.55571,1.0,1.0,1.0,1.0 +hybrid+rerank,577,10,2455.85375,0.0,0.0,0.0,0.0 +hybrid+rerank,578,10,2247.41279,1.0,1.0,1.0,1.0 +hybrid+rerank,587,10,2273.81004,0.63093,1.0,0.5,1.0 +hybrid+rerank,589,10,2935.85592,1.0,1.0,1.0,1.0 +hybrid+rerank,593,10,2625.85046,1.0,1.0,1.0,1.0 +hybrid+rerank,597,10,2303.22287,0.83855,1.0,1.0,1.0 +hybrid+rerank,598,10,2016.46425,0.33333,1.0,0.14286,0.0 +hybrid+rerank,613,10,2231.54079,1.0,1.0,1.0,1.0 +hybrid+rerank,619,10,2507.72154,0.38685,0.5,0.5,1.0 +hybrid+rerank,623,10,1581.63175,0.0,0.0,0.0,0.0 +hybrid+rerank,628,10,2037.66483,1.0,1.0,1.0,1.0 +hybrid+rerank,636,10,2228.13596,1.0,1.0,1.0,1.0 +hybrid+rerank,637,10,2476.14879,0.63093,1.0,0.5,1.0 +hybrid+rerank,641,10,2276.32887,0.79039,1.0,1.0,1.0 +hybrid+rerank,644,10,2409.95633,1.0,1.0,1.0,1.0 +hybrid+rerank,649,10,2043.13167,1.0,1.0,1.0,1.0 +hybrid+rerank,659,10,2077.06783,0.30103,1.0,0.11111,0.0 +hybrid+rerank,660,10,1774.17813,0.0,0.0,0.0,0.0 +hybrid+rerank,674,10,1613.86021,1.0,1.0,1.0,1.0 +hybrid+rerank,684,10,1769.95929,0.43068,1.0,0.25,0.0 +hybrid+rerank,690,10,2445.19929,1.0,1.0,1.0,1.0 +hybrid+rerank,691,10,2074.64933,0.63093,1.0,0.5,1.0 +hybrid+rerank,692,10,1936.51538,1.0,1.0,1.0,1.0 +hybrid+rerank,693,10,1863.65142,1.0,1.0,1.0,1.0 +hybrid+rerank,700,10,1935.88317,0.31546,1.0,0.125,0.0 +hybrid+rerank,702,10,1712.45196,0.31546,1.0,0.125,0.0 +hybrid+rerank,715,10,2186.99967,0.0,0.0,0.0,0.0 +hybrid+rerank,716,10,2120.30608,0.0,0.0,0.0,0.0 +hybrid+rerank,718,10,1907.84733,0.63093,1.0,0.5,1.0 +hybrid+rerank,721,10,2085.07854,1.0,1.0,1.0,1.0 +hybrid+rerank,723,10,2007.05979,1.0,1.0,1.0,1.0 +hybrid+rerank,727,10,1804.63654,0.30103,1.0,0.11111,0.0 +hybrid+rerank,728,10,1827.18792,0.80657,1.0,1.0,1.0 +hybrid+rerank,729,10,1747.24667,1.0,1.0,1.0,1.0 +hybrid+rerank,742,10,1882.26938,1.0,1.0,1.0,1.0 +hybrid+rerank,743,10,1857.72092,1.0,1.0,1.0,1.0 +hybrid+rerank,744,10,1522.60883,0.63093,1.0,0.5,1.0 +hybrid+rerank,756,10,1484.56021,0.43068,1.0,0.25,0.0 +hybrid+rerank,759,10,1536.53958,1.0,1.0,1.0,1.0 +hybrid+rerank,768,10,1997.31046,0.0,0.0,0.0,0.0 +hybrid+rerank,770,10,1618.14188,1.0,1.0,1.0,1.0 +hybrid+rerank,775,10,1593.61771,0.31546,1.0,0.125,0.0 +hybrid+rerank,781,10,1685.83188,0.63093,1.0,0.5,1.0 +hybrid+rerank,783,10,1751.09017,0.0,0.0,0.0,0.0 +hybrid+rerank,784,10,1653.52717,1.0,1.0,1.0,1.0 +hybrid+rerank,785,10,1770.60642,0.31546,1.0,0.125,0.0 +hybrid+rerank,793,10,1598.17408,1.0,1.0,1.0,1.0 +hybrid+rerank,800,10,1442.85079,0.63093,1.0,0.5,1.0 +hybrid+rerank,805,10,1699.55029,1.0,1.0,1.0,1.0 +hybrid+rerank,808,10,1759.56121,1.0,1.0,1.0,1.0 +hybrid+rerank,811,10,1900.56683,1.0,1.0,1.0,1.0 +hybrid+rerank,814,10,1767.01379,1.0,1.0,1.0,1.0 +hybrid+rerank,820,10,1698.33738,0.31546,1.0,0.125,0.0 +hybrid+rerank,821,10,1801.30929,0.35621,1.0,0.16667,0.0 +hybrid+rerank,823,10,1753.02288,1.0,1.0,1.0,1.0 +hybrid+rerank,830,10,1673.44437,0.0,0.0,0.0,0.0 +hybrid+rerank,831,10,1820.71071,0.0,0.0,0.0,0.0 +hybrid+rerank,832,10,1794.33362,1.0,1.0,1.0,1.0 +hybrid+rerank,834,10,1822.03171,0.35621,1.0,0.16667,0.0 +hybrid+rerank,837,10,1744.40125,0.63093,1.0,0.5,1.0 +hybrid+rerank,839,10,1564.36408,1.0,1.0,1.0,1.0 +hybrid+rerank,845,10,1332.10967,1.0,1.0,1.0,1.0 +hybrid+rerank,847,10,1372.99842,1.0,1.0,1.0,1.0 +hybrid+rerank,852,10,1859.338,0.63093,1.0,0.5,1.0 +hybrid+rerank,859,10,1731.81333,0.5,1.0,0.33333,1.0 +hybrid+rerank,870,10,1807.66912,0.0,0.0,0.0,0.0 +hybrid+rerank,873,10,1815.19546,0.25202,0.4,0.2,0.0 +hybrid+rerank,879,10,1918.41608,1.0,1.0,1.0,1.0 +hybrid+rerank,880,10,1827.93067,1.0,1.0,1.0,1.0 +hybrid+rerank,882,10,1764.71237,1.0,1.0,1.0,1.0 +hybrid+rerank,887,10,1920.34763,0.0,0.0,0.0,0.0 +hybrid+rerank,903,10,1694.20171,1.0,1.0,1.0,1.0 +hybrid+rerank,904,10,1727.3885,1.0,1.0,1.0,1.0 +hybrid+rerank,907,10,1712.83708,1.0,1.0,1.0,1.0 +hybrid+rerank,911,10,1700.4055,1.0,1.0,1.0,1.0 +hybrid+rerank,913,10,1675.09887,0.0,0.0,0.0,0.0 +hybrid+rerank,914,10,1851.64708,0.0,0.0,0.0,0.0 +hybrid+rerank,921,10,1666.46967,1.0,1.0,1.0,1.0 +hybrid+rerank,922,10,1895.99621,1.0,1.0,1.0,1.0 +hybrid+rerank,936,10,1728.94858,1.0,1.0,1.0,1.0 +hybrid+rerank,956,10,1734.83846,1.0,1.0,1.0,1.0 +hybrid+rerank,957,10,1182.0245,1.0,1.0,1.0,1.0 +hybrid+rerank,960,10,1782.16104,1.0,1.0,1.0,1.0 +hybrid+rerank,967,10,1855.27217,0.91972,1.0,1.0,1.0 +hybrid+rerank,971,10,1420.56483,0.95583,1.0,1.0,1.0 +hybrid+rerank,975,10,1816.30175,0.0,0.0,0.0,0.0 +hybrid+rerank,982,10,1737.56396,1.0,1.0,1.0,1.0 +hybrid+rerank,985,10,1745.89242,1.0,1.0,1.0,1.0 +hybrid+rerank,993,10,1534.03237,1.0,1.0,1.0,1.0 +hybrid+rerank,1012,10,1776.00021,1.0,1.0,1.0,1.0 +hybrid+rerank,1014,10,1801.69492,1.0,1.0,1.0,1.0 +hybrid+rerank,1019,10,1676.65617,0.63093,1.0,0.5,1.0 +hybrid+rerank,1020,10,1847.7215,0.5,1.0,0.33333,1.0 +hybrid+rerank,1021,10,1750.02842,0.5,1.0,0.33333,1.0 +hybrid+rerank,1024,10,1850.81383,1.0,1.0,1.0,1.0 +hybrid+rerank,1029,10,1698.81321,0.76536,0.66667,1.0,1.0 +hybrid+rerank,1041,10,1530.16858,0.21841,0.5,0.16667,0.0 +hybrid+rerank,1049,10,1769.188,0.43068,1.0,0.25,0.0 +hybrid+rerank,1062,10,1655.10662,1.0,1.0,1.0,1.0 +hybrid+rerank,1086,10,2278.26263,1.0,1.0,1.0,1.0 +hybrid+rerank,1088,10,1869.96967,1.0,1.0,1.0,1.0 +hybrid+rerank,1089,10,1770.29996,1.0,1.0,1.0,1.0 +hybrid+rerank,1099,10,1635.02862,0.43068,1.0,0.25,0.0 +hybrid+rerank,1100,10,1373.6825,0.43068,1.0,0.25,0.0 +hybrid+rerank,1104,10,1932.53958,1.0,1.0,1.0,1.0 +hybrid+rerank,1107,10,1458.60487,1.0,1.0,1.0,1.0 +hybrid+rerank,1110,10,1994.51983,0.0,0.0,0.0,0.0 +hybrid+rerank,1121,10,1604.53483,1.0,1.0,1.0,1.0 +hybrid+rerank,1130,10,1716.95071,1.0,1.0,1.0,1.0 +hybrid+rerank,1132,10,1543.30967,0.61315,0.5,1.0,1.0 +hybrid+rerank,1137,10,1721.85433,1.0,1.0,1.0,1.0 +hybrid+rerank,1140,10,1587.15683,1.0,1.0,1.0,1.0 +hybrid+rerank,1144,10,1782.209,1.0,1.0,1.0,1.0 +hybrid+rerank,1146,10,1665.93671,1.0,1.0,1.0,1.0 +hybrid+rerank,1150,10,1483.19104,1.0,1.0,1.0,1.0 +hybrid+rerank,1163,10,1881.23754,1.0,1.0,1.0,1.0 +hybrid+rerank,1175,10,1854.49079,0.0,0.0,0.0,0.0 +hybrid+rerank,1179,10,1850.91433,0.28906,1.0,0.1,0.0 +hybrid+rerank,1180,10,1563.48508,1.0,1.0,1.0,1.0 +hybrid+rerank,1185,10,2208.27717,1.0,1.0,1.0,1.0 +hybrid+rerank,1187,10,1820.66921,1.0,1.0,1.0,1.0 +hybrid+rerank,1191,10,1750.99079,0.0,0.0,0.0,0.0 +hybrid+rerank,1194,10,1756.85483,1.0,1.0,1.0,1.0 +hybrid+rerank,1196,10,1804.89567,0.43068,1.0,0.25,0.0 +hybrid+rerank,1197,10,1862.993,0.43068,1.0,0.25,0.0 +hybrid+rerank,1199,10,1922.86983,0.0,0.0,0.0,0.0 +hybrid+rerank,1200,10,1869.19333,0.0,0.0,0.0,0.0 +hybrid+rerank,1202,10,1773.03513,0.63093,1.0,0.5,1.0 +hybrid+rerank,1204,10,1828.65233,0.43068,1.0,0.25,0.0 +hybrid+rerank,1207,10,1936.47112,1.0,1.0,1.0,1.0 +hybrid+rerank,1213,10,1658.5785,0.0,0.0,0.0,0.0 +hybrid+rerank,1216,10,1854.24917,1.0,1.0,1.0,1.0 +hybrid+rerank,1221,10,1843.88021,0.31546,1.0,0.125,0.0 +hybrid+rerank,1225,10,1856.49437,1.0,1.0,1.0,1.0 +hybrid+rerank,1226,10,1861.14942,0.0,0.0,0.0,0.0 +hybrid+rerank,1232,10,1824.37296,1.0,1.0,1.0,1.0 +hybrid+rerank,1241,10,1764.23654,1.0,1.0,1.0,1.0 +hybrid+rerank,1245,10,1843.84146,0.0,0.0,0.0,0.0 +hybrid+rerank,1259,10,2329.79592,1.0,1.0,1.0,1.0 +hybrid+rerank,1262,10,1625.94092,1.0,1.0,1.0,1.0 +hybrid+rerank,1266,10,1547.18692,0.5,1.0,0.33333,1.0 +hybrid+rerank,1270,10,1900.91437,1.0,1.0,1.0,1.0 +hybrid+rerank,1271,10,1570.34525,1.0,1.0,1.0,1.0 +hybrid+rerank,1272,10,1782.94254,0.5,1.0,0.33333,1.0 +hybrid+rerank,1273,10,1572.64512,1.0,1.0,1.0,1.0 +hybrid+rerank,1274,10,1684.64779,0.93252,1.0,1.0,1.0 +hybrid+rerank,1278,10,1950.26783,0.0,0.0,0.0,0.0 +hybrid+rerank,1279,10,1963.82242,0.28906,1.0,0.1,0.0 +hybrid+rerank,1280,10,1683.02087,0.0,0.0,0.0,0.0 +hybrid+rerank,1281,10,1976.83963,0.0,0.0,0.0,0.0 +hybrid+rerank,1282,10,1859.34846,1.0,1.0,1.0,1.0 +hybrid+rerank,1290,10,1747.40112,1.0,1.0,1.0,1.0 +hybrid+rerank,1292,10,1879.66133,0.5,1.0,0.33333,1.0 +hybrid+rerank,1298,10,2156.0955,1.0,1.0,1.0,1.0 +hybrid+rerank,1303,10,1889.93742,1.0,1.0,1.0,1.0 +hybrid+rerank,1316,10,1541.00171,0.0,0.0,0.0,0.0 +hybrid+rerank,1319,10,1803.38375,1.0,1.0,1.0,1.0 +hybrid+rerank,1320,10,1657.86408,1.0,1.0,1.0,1.0 +hybrid+rerank,1332,10,1844.11908,0.0,0.0,0.0,0.0 +hybrid+rerank,1335,10,1678.57638,1.0,1.0,1.0,1.0 +hybrid+rerank,1336,10,1585.17892,1.0,1.0,1.0,1.0 +hybrid+rerank,1337,10,1608.24063,0.63093,1.0,0.5,1.0 +hybrid+rerank,1339,10,1718.83742,1.0,1.0,1.0,1.0 +hybrid+rerank,1344,10,1779.88954,0.5,1.0,0.33333,1.0 +hybrid+rerank,1352,10,1797.83321,1.0,1.0,1.0,1.0 +hybrid+rerank,1359,10,2114.56729,1.0,1.0,1.0,1.0 +hybrid+rerank,1362,10,1716.04108,0.31546,1.0,0.125,0.0 +hybrid+rerank,1363,10,1626.20363,0.5,1.0,0.33333,1.0 +hybrid+rerank,1368,10,1421.02004,1.0,1.0,1.0,1.0 +hybrid+rerank,1370,10,1654.39704,0.63093,1.0,0.5,1.0 +hybrid+rerank,1379,10,1773.18917,0.83268,1.0,1.0,1.0 +hybrid+rerank,1382,10,1568.42992,1.0,1.0,1.0,1.0 +hybrid+rerank,1385,10,1868.90092,1.0,1.0,1.0,1.0 +hybrid+rerank,1389,10,1911.41483,1.0,1.0,1.0,1.0 +hybrid+rerank,1395,10,1770.696,1.0,1.0,1.0,1.0 +app_default,1,0,801.06154,0.0,0.0,0.0,0.0 +app_default,3,3,557.1665,1.0,1.0,1.0,1.0 +app_default,5,1,727.94633,1.0,1.0,1.0,1.0 +app_default,13,2,596.87787,0.0,0.0,0.0,0.0 +app_default,36,3,469.03271,0.0,0.0,0.0,0.0 +app_default,42,1,548.51225,1.0,1.0,1.0,1.0 +app_default,48,2,704.04546,0.0,0.0,0.0,0.0 +app_default,49,2,550.37642,1.0,1.0,1.0,1.0 +app_default,50,3,382.0185,1.0,1.0,1.0,1.0 +app_default,51,3,752.71096,0.63093,1.0,0.5,1.0 +app_default,53,3,661.30746,1.0,1.0,1.0,1.0 +app_default,54,3,555.41817,0.0,0.0,0.0,0.0 +app_default,56,2,682.91017,1.0,1.0,1.0,1.0 +app_default,57,2,739.57292,1.0,1.0,1.0,1.0 +app_default,70,3,749.05862,0.0,0.0,0.0,0.0 +app_default,72,0,720.43308,0.0,0.0,0.0,0.0 +app_default,75,1,632.96217,1.0,1.0,1.0,1.0 +app_default,94,0,747.15163,0.0,0.0,0.0,0.0 +app_default,99,1,706.50179,1.0,1.0,1.0,1.0 +app_default,100,2,374.21871,1.0,1.0,1.0,1.0 +app_default,113,3,585.72417,1.0,1.0,1.0,1.0 +app_default,115,1,491.00208,1.0,1.0,1.0,1.0 +app_default,118,3,539.08554,1.0,1.0,1.0,1.0 +app_default,124,3,647.60271,1.0,1.0,1.0,1.0 +app_default,127,2,657.96746,0.63093,1.0,0.5,1.0 +app_default,128,0,742.27125,0.0,0.0,0.0,0.0 +app_default,129,1,669.28846,1.0,1.0,1.0,1.0 +app_default,130,1,677.23038,1.0,1.0,1.0,1.0 +app_default,132,3,611.48596,0.0,0.0,0.0,0.0 +app_default,133,2,770.24463,0.55315,0.4,1.0,1.0 +app_default,137,1,651.29367,1.0,1.0,1.0,1.0 +app_default,141,3,528.11987,0.91972,1.0,1.0,1.0 +app_default,142,3,795.56271,1.0,1.0,1.0,1.0 +app_default,143,3,784.17713,1.0,1.0,1.0,1.0 +app_default,146,3,746.43846,1.0,1.0,1.0,1.0 +app_default,148,3,578.36792,1.0,1.0,1.0,1.0 +app_default,163,3,489.13033,0.63093,1.0,0.5,1.0 +app_default,171,3,412.60733,1.0,1.0,1.0,1.0 +app_default,179,3,593.38338,0.63668,0.5,1.0,1.0 +app_default,180,2,744.47083,1.0,1.0,1.0,1.0 +app_default,183,3,487.39442,1.0,1.0,1.0,1.0 +app_default,185,3,787.57225,0.5,1.0,0.33333,1.0 +app_default,198,0,813.12908,0.0,0.0,0.0,0.0 +app_default,208,3,750.67367,1.0,1.0,1.0,1.0 +app_default,212,3,366.35787,0.5,1.0,0.33333,1.0 +app_default,213,3,562.39692,0.0,0.0,0.0,0.0 +app_default,216,3,665.56942,1.0,1.0,1.0,1.0 +app_default,217,3,745.48637,1.0,1.0,1.0,1.0 +app_default,218,3,609.44096,1.0,1.0,1.0,1.0 +app_default,219,3,655.25525,1.0,1.0,1.0,1.0 +app_default,230,0,667.2805,0.0,0.0,0.0,0.0 +app_default,232,3,539.26321,0.63093,1.0,0.5,1.0 +app_default,233,1,447.91442,1.0,1.0,1.0,1.0 +app_default,236,1,491.73383,1.0,1.0,1.0,1.0 +app_default,237,3,562.98529,0.0,0.0,0.0,0.0 +app_default,238,1,453.87163,0.0,0.0,0.0,0.0 +app_default,239,1,384.43038,0.0,0.0,0.0,0.0 +app_default,248,1,694.11996,1.0,1.0,1.0,1.0 +app_default,249,1,742.63271,1.0,1.0,1.0,1.0 +app_default,261,3,632.17854,0.38685,0.5,0.5,1.0 +app_default,268,2,591.79908,0.63093,1.0,0.5,1.0 +app_default,269,1,420.38621,0.0,0.0,0.0,0.0 +app_default,274,3,520.99658,1.0,1.0,1.0,1.0 +app_default,275,3,645.28904,0.70392,0.66667,1.0,1.0 +app_default,279,3,581.5815,1.0,1.0,1.0,1.0 +app_default,294,2,485.19783,1.0,1.0,1.0,1.0 +app_default,295,3,407.44808,1.0,1.0,1.0,1.0 +app_default,298,3,725.65512,1.0,1.0,1.0,1.0 +app_default,300,3,470.20321,0.5,1.0,0.33333,1.0 +app_default,303,2,742.37092,0.0,0.0,0.0,0.0 +app_default,312,2,602.00496,0.0,0.0,0.0,0.0 +app_default,314,2,760.31371,0.0,0.0,0.0,0.0 +app_default,324,1,697.57546,0.0,0.0,0.0,0.0 +app_default,327,2,718.37479,1.0,1.0,1.0,1.0 +app_default,338,3,660.32063,1.0,1.0,1.0,1.0 +app_default,343,3,716.56871,0.30657,0.5,0.33333,1.0 +app_default,350,2,740.199,1.0,1.0,1.0,1.0 +app_default,354,2,399.49321,1.0,1.0,1.0,1.0 +app_default,362,3,621.64717,1.0,1.0,1.0,1.0 +app_default,380,2,787.98275,1.0,1.0,1.0,1.0 +app_default,384,2,684.36313,0.0,0.0,0.0,0.0 +app_default,385,3,643.50021,0.38685,0.5,0.5,1.0 +app_default,386,1,722.01125,1.0,1.0,1.0,1.0 +app_default,388,0,670.964,0.0,0.0,0.0,0.0 +app_default,399,3,515.6895,1.0,1.0,1.0,1.0 +app_default,410,3,568.26125,1.0,1.0,1.0,1.0 +app_default,411,3,530.74896,1.0,1.0,1.0,1.0 +app_default,415,3,568.44654,0.63093,1.0,0.5,1.0 +app_default,421,0,830.41217,0.0,0.0,0.0,0.0 +app_default,431,3,715.14292,0.0,0.0,0.0,0.0 +app_default,436,1,662.69879,1.0,1.0,1.0,1.0 +app_default,437,3,639.35867,0.0,0.0,0.0,0.0 +app_default,439,2,691.17225,1.0,1.0,1.0,1.0 +app_default,440,1,723.24533,1.0,1.0,1.0,1.0 +app_default,443,3,769.70079,1.0,1.0,1.0,1.0 +app_default,452,3,713.53529,0.38685,0.5,0.5,1.0 +app_default,475,0,715.78967,0.0,0.0,0.0,0.0 +app_default,478,1,476.36446,1.0,1.0,1.0,1.0 +app_default,491,1,765.38258,1.0,1.0,1.0,1.0 +app_default,501,2,729.84696,1.0,1.0,1.0,1.0 +app_default,502,0,855.16125,0.0,0.0,0.0,0.0 +app_default,507,3,705.73892,0.63093,1.0,0.5,1.0 +app_default,508,1,697.68054,0.0,0.0,0.0,0.0 +app_default,513,3,725.21671,0.63093,1.0,0.5,1.0 +app_default,514,3,663.06433,0.5,1.0,0.33333,1.0 +app_default,516,3,571.57404,1.0,1.0,1.0,1.0 +app_default,517,2,814.80042,0.0,0.0,0.0,0.0 +app_default,521,1,607.09921,1.0,1.0,1.0,1.0 +app_default,525,3,621.29271,1.0,1.0,1.0,1.0 +app_default,527,2,729.59213,0.0,0.0,0.0,0.0 +app_default,528,3,731.35717,0.5,1.0,0.33333,1.0 +app_default,532,1,666.19075,1.0,1.0,1.0,1.0 +app_default,533,1,675.88763,1.0,1.0,1.0,1.0 +app_default,535,3,699.86533,0.0,0.0,0.0,0.0 +app_default,536,1,696.32813,1.0,1.0,1.0,1.0 +app_default,539,3,645.96317,1.0,1.0,1.0,1.0 +app_default,540,3,706.93013,0.30657,0.5,0.33333,1.0 +app_default,544,2,766.09117,0.0,0.0,0.0,0.0 +app_default,549,3,722.05942,1.0,1.0,1.0,1.0 +app_default,551,2,629.60562,0.0,0.0,0.0,0.0 +app_default,552,2,409.37154,1.0,1.0,1.0,1.0 +app_default,554,3,632.19633,0.0,0.0,0.0,0.0 +app_default,560,3,730.29488,0.0,0.0,0.0,0.0 +app_default,569,3,696.36696,1.0,1.0,1.0,1.0 +app_default,575,1,503.04192,1.0,1.0,1.0,1.0 +app_default,577,0,776.57404,0.0,0.0,0.0,0.0 +app_default,578,3,799.13821,1.0,1.0,1.0,1.0 +app_default,587,1,814.46058,0.0,0.0,0.0,0.0 +app_default,589,1,734.74488,1.0,1.0,1.0,1.0 +app_default,593,2,647.64767,1.0,1.0,1.0,1.0 +app_default,597,3,544.41475,0.46928,0.33333,1.0,1.0 +app_default,598,3,537.15992,0.0,0.0,0.0,0.0 +app_default,613,1,706.86562,1.0,1.0,1.0,1.0 +app_default,619,0,875.42712,0.0,0.0,0.0,0.0 +app_default,623,3,682.35742,0.0,0.0,0.0,0.0 +app_default,628,3,616.01708,1.0,1.0,1.0,1.0 +app_default,636,3,619.34737,1.0,1.0,1.0,1.0 +app_default,637,1,806.97996,0.0,0.0,0.0,0.0 +app_default,641,2,723.93321,0.61315,0.5,1.0,1.0 +app_default,644,2,734.36533,1.0,1.0,1.0,1.0 +app_default,649,0,769.13254,0.0,0.0,0.0,0.0 +app_default,659,0,828.51421,0.0,0.0,0.0,0.0 +app_default,660,0,770.14575,0.0,0.0,0.0,0.0 +app_default,674,3,612.53183,1.0,1.0,1.0,1.0 +app_default,684,3,436.396,0.0,0.0,0.0,0.0 +app_default,690,0,803.73075,0.0,0.0,0.0,0.0 +app_default,691,3,722.5025,0.63093,1.0,0.5,1.0 +app_default,692,1,700.62571,1.0,1.0,1.0,1.0 +app_default,693,1,733.76229,1.0,1.0,1.0,1.0 +app_default,700,1,726.84675,0.0,0.0,0.0,0.0 +app_default,702,2,632.58379,0.0,0.0,0.0,0.0 +app_default,715,3,876.20154,0.0,0.0,0.0,0.0 +app_default,716,2,750.60408,0.0,0.0,0.0,0.0 +app_default,718,0,721.71987,0.0,0.0,0.0,0.0 +app_default,721,1,666.54358,1.0,1.0,1.0,1.0 +app_default,723,1,377.67925,1.0,1.0,1.0,1.0 +app_default,727,2,745.71896,0.0,0.0,0.0,0.0 +app_default,728,3,703.06908,0.61315,0.5,1.0,1.0 +app_default,729,3,793.64742,1.0,1.0,1.0,1.0 +app_default,742,2,697.51779,1.0,1.0,1.0,1.0 +app_default,743,2,782.47058,1.0,1.0,1.0,1.0 +app_default,744,3,364.24446,0.63093,1.0,0.5,1.0 +app_default,756,3,383.12058,0.5,1.0,0.33333,1.0 +app_default,759,3,561.73392,1.0,1.0,1.0,1.0 +app_default,768,1,824.63696,0.0,0.0,0.0,0.0 +app_default,770,3,747.14338,1.0,1.0,1.0,1.0 +app_default,775,3,361.18154,0.0,0.0,0.0,0.0 +app_default,781,3,411.9885,0.63093,1.0,0.5,1.0 +app_default,783,3,623.65679,0.0,0.0,0.0,0.0 +app_default,784,3,603.592,1.0,1.0,1.0,1.0 +app_default,785,1,744.82283,0.0,0.0,0.0,0.0 +app_default,793,2,688.56838,1.0,1.0,1.0,1.0 +app_default,800,3,668.10437,0.63093,1.0,0.5,1.0 +app_default,805,3,637.29729,1.0,1.0,1.0,1.0 +app_default,808,1,752.20942,1.0,1.0,1.0,1.0 +app_default,811,3,786.6,1.0,1.0,1.0,1.0 +app_default,814,3,827.68921,1.0,1.0,1.0,1.0 +app_default,820,0,672.36662,0.0,0.0,0.0,0.0 +app_default,821,0,774.35471,0.0,0.0,0.0,0.0 +app_default,823,3,617.26958,1.0,1.0,1.0,1.0 +app_default,830,1,710.89792,0.0,0.0,0.0,0.0 +app_default,831,0,676.51779,0.0,0.0,0.0,0.0 +app_default,832,3,675.29442,1.0,1.0,1.0,1.0 +app_default,834,1,466.77192,0.0,0.0,0.0,0.0 +app_default,837,3,710.97429,0.63093,1.0,0.5,1.0 +app_default,839,3,592.71854,1.0,1.0,1.0,1.0 +app_default,845,3,612.36712,1.0,1.0,1.0,1.0 +app_default,847,2,637.856,1.0,1.0,1.0,1.0 +app_default,852,3,629.98496,0.63093,1.0,0.5,1.0 +app_default,859,3,683.4885,0.5,1.0,0.33333,1.0 +app_default,870,3,740.46829,0.0,0.0,0.0,0.0 +app_default,873,3,886.597,0.0,0.0,0.0,0.0 +app_default,879,1,719.76833,1.0,1.0,1.0,1.0 +app_default,880,1,721.69313,1.0,1.0,1.0,1.0 +app_default,882,1,741.74371,1.0,1.0,1.0,1.0 +app_default,887,0,742.558,0.0,0.0,0.0,0.0 +app_default,903,3,626.51775,1.0,1.0,1.0,1.0 +app_default,904,2,750.49962,1.0,1.0,1.0,1.0 +app_default,907,3,654.22083,1.0,1.0,1.0,1.0 +app_default,911,3,778.22804,1.0,1.0,1.0,1.0 +app_default,913,3,674.57133,0.0,0.0,0.0,0.0 +app_default,914,3,738.47133,0.0,0.0,0.0,0.0 +app_default,921,3,553.41679,1.0,1.0,1.0,1.0 +app_default,922,1,666.22158,1.0,1.0,1.0,1.0 +app_default,936,1,707.23279,1.0,1.0,1.0,1.0 +app_default,956,3,709.88983,1.0,1.0,1.0,1.0 +app_default,957,1,377.1225,1.0,1.0,1.0,1.0 +app_default,960,1,772.31379,1.0,1.0,1.0,1.0 +app_default,967,3,743.94733,0.91972,1.0,1.0,1.0 +app_default,971,3,404.00642,0.63668,0.5,1.0,1.0 +app_default,975,3,842.59383,0.0,0.0,0.0,0.0 +app_default,982,1,847.71079,1.0,1.0,1.0,1.0 +app_default,985,3,655.27313,1.0,1.0,1.0,1.0 +app_default,993,3,532.23196,1.0,1.0,1.0,1.0 +app_default,1012,3,542.77804,1.0,1.0,1.0,1.0 +app_default,1014,1,751.30554,1.0,1.0,1.0,1.0 +app_default,1019,2,737.36163,0.63093,1.0,0.5,1.0 +app_default,1020,2,649.38075,0.63093,1.0,0.5,1.0 +app_default,1021,1,612.17242,0.0,0.0,0.0,0.0 +app_default,1024,2,736.76096,1.0,1.0,1.0,1.0 +app_default,1029,3,732.44637,0.76536,0.66667,1.0,1.0 +app_default,1041,3,382.74017,0.0,0.0,0.0,0.0 +app_default,1049,0,735.07508,0.0,0.0,0.0,0.0 +app_default,1062,3,433.46029,1.0,1.0,1.0,1.0 +app_default,1086,1,760.7645,1.0,1.0,1.0,1.0 +app_default,1088,3,775.32679,1.0,1.0,1.0,1.0 +app_default,1089,2,463.02262,1.0,1.0,1.0,1.0 +app_default,1099,3,565.74225,0.0,0.0,0.0,0.0 +app_default,1100,3,647.59862,0.0,0.0,0.0,0.0 +app_default,1104,2,714.64467,1.0,1.0,1.0,1.0 +app_default,1107,3,519.76433,1.0,1.0,1.0,1.0 +app_default,1110,0,844.30404,0.0,0.0,0.0,0.0 +app_default,1121,3,656.1505,1.0,1.0,1.0,1.0 +app_default,1130,3,681.04171,1.0,1.0,1.0,1.0 +app_default,1132,3,601.85658,0.61315,0.5,1.0,1.0 +app_default,1137,3,649.20079,1.0,1.0,1.0,1.0 +app_default,1140,0,839.9995,0.0,0.0,0.0,0.0 +app_default,1144,2,569.78329,1.0,1.0,1.0,1.0 +app_default,1146,1,744.56688,1.0,1.0,1.0,1.0 +app_default,1150,3,786.81492,1.0,1.0,1.0,1.0 +app_default,1163,2,756.89904,1.0,1.0,1.0,1.0 +app_default,1175,0,838.89579,0.0,0.0,0.0,0.0 +app_default,1179,1,846.98142,0.0,0.0,0.0,0.0 +app_default,1180,3,781.76379,1.0,1.0,1.0,1.0 +app_default,1185,1,717.31479,1.0,1.0,1.0,1.0 +app_default,1187,3,736.63338,1.0,1.0,1.0,1.0 +app_default,1191,0,851.69958,0.0,0.0,0.0,0.0 +app_default,1194,1,431.51337,1.0,1.0,1.0,1.0 +app_default,1196,0,375.89058,0.0,0.0,0.0,0.0 +app_default,1197,0,383.41571,0.0,0.0,0.0,0.0 +app_default,1199,2,707.76188,0.0,0.0,0.0,0.0 +app_default,1200,0,881.42583,0.0,0.0,0.0,0.0 +app_default,1202,3,609.41867,0.63093,1.0,0.5,1.0 +app_default,1204,1,759.772,0.0,0.0,0.0,0.0 +app_default,1207,1,717.81033,1.0,1.0,1.0,1.0 +app_default,1213,3,656.63254,0.0,0.0,0.0,0.0 +app_default,1216,1,854.88125,1.0,1.0,1.0,1.0 +app_default,1221,0,824.49142,0.0,0.0,0.0,0.0 +app_default,1225,2,847.90271,1.0,1.0,1.0,1.0 +app_default,1226,1,827.4865,0.0,0.0,0.0,0.0 +app_default,1232,1,628.81938,1.0,1.0,1.0,1.0 +app_default,1241,3,849.86767,1.0,1.0,1.0,1.0 +app_default,1245,0,664.81954,0.0,0.0,0.0,0.0 +app_default,1259,3,675.03279,1.0,1.0,1.0,1.0 +app_default,1262,3,366.21554,1.0,1.0,1.0,1.0 +app_default,1266,3,611.48925,0.5,1.0,0.33333,1.0 +app_default,1270,1,710.46613,1.0,1.0,1.0,1.0 +app_default,1271,1,605.10258,1.0,1.0,1.0,1.0 +app_default,1272,0,720.37088,0.0,0.0,0.0,0.0 +app_default,1273,3,734.29146,1.0,1.0,1.0,1.0 +app_default,1274,3,600.94825,0.76536,0.66667,1.0,1.0 +app_default,1278,1,826.76212,0.0,0.0,0.0,0.0 +app_default,1279,1,859.31325,0.0,0.0,0.0,0.0 +app_default,1280,3,749.13333,0.0,0.0,0.0,0.0 +app_default,1281,1,742.94087,0.0,0.0,0.0,0.0 +app_default,1282,1,779.33721,1.0,1.0,1.0,1.0 +app_default,1290,1,677.76404,1.0,1.0,1.0,1.0 +app_default,1292,3,817.708,0.63093,1.0,0.5,1.0 +app_default,1298,1,714.21529,1.0,1.0,1.0,1.0 +app_default,1303,0,736.85917,0.0,0.0,0.0,0.0 +app_default,1316,1,524.80337,0.0,0.0,0.0,0.0 +app_default,1319,3,749.07271,1.0,1.0,1.0,1.0 +app_default,1320,3,604.04442,1.0,1.0,1.0,1.0 +app_default,1332,3,782.65513,0.0,0.0,0.0,0.0 +app_default,1335,3,604.14533,1.0,1.0,1.0,1.0 +app_default,1336,3,746.17054,1.0,1.0,1.0,1.0 +app_default,1337,3,790.92975,0.63093,1.0,0.5,1.0 +app_default,1339,1,787.45567,1.0,1.0,1.0,1.0 +app_default,1344,3,723.88013,0.63093,1.0,0.5,1.0 +app_default,1352,2,643.87738,1.0,1.0,1.0,1.0 +app_default,1359,3,631.70492,1.0,1.0,1.0,1.0 +app_default,1362,0,810.05892,0.0,0.0,0.0,0.0 +app_default,1363,0,822.25287,0.0,0.0,0.0,0.0 +app_default,1368,3,590.85192,0.0,0.0,0.0,0.0 +app_default,1370,2,674.61442,0.63093,1.0,0.5,1.0 +app_default,1379,3,650.42096,0.39038,0.25,1.0,1.0 +app_default,1382,3,667.65258,1.0,1.0,1.0,1.0 +app_default,1385,2,784.91833,1.0,1.0,1.0,1.0 +app_default,1389,3,816.94883,1.0,1.0,1.0,1.0 +app_default,1395,0,687.10367,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1,3,819.21413,0.0,0.0,0.0,0.0 +app_default_no_cutoff,3,3,595.71054,1.0,1.0,1.0,1.0 +app_default_no_cutoff,5,3,763.79496,1.0,1.0,1.0,1.0 +app_default_no_cutoff,13,3,624.25687,0.0,0.0,0.0,0.0 +app_default_no_cutoff,36,3,540.99713,0.0,0.0,0.0,0.0 +app_default_no_cutoff,42,3,584.25958,1.0,1.0,1.0,1.0 +app_default_no_cutoff,48,3,755.18192,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,49,3,632.31117,1.0,1.0,1.0,1.0 +app_default_no_cutoff,50,3,383.70846,1.0,1.0,1.0,1.0 +app_default_no_cutoff,51,3,771.36158,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,53,3,768.36725,1.0,1.0,1.0,1.0 +app_default_no_cutoff,54,3,601.12975,0.0,0.0,0.0,0.0 +app_default_no_cutoff,56,3,809.05183,1.0,1.0,1.0,1.0 +app_default_no_cutoff,57,3,774.18883,1.0,1.0,1.0,1.0 +app_default_no_cutoff,70,3,823.82108,0.0,0.0,0.0,0.0 +app_default_no_cutoff,72,3,867.03817,1.0,1.0,1.0,1.0 +app_default_no_cutoff,75,3,667.28038,1.0,1.0,1.0,1.0 +app_default_no_cutoff,94,3,821.55833,0.0,0.0,0.0,0.0 +app_default_no_cutoff,99,3,732.26254,1.0,1.0,1.0,1.0 +app_default_no_cutoff,100,3,371.56737,1.0,1.0,1.0,1.0 +app_default_no_cutoff,113,3,650.6335,1.0,1.0,1.0,1.0 +app_default_no_cutoff,115,3,490.01892,1.0,1.0,1.0,1.0 +app_default_no_cutoff,118,3,523.34171,1.0,1.0,1.0,1.0 +app_default_no_cutoff,124,3,655.01625,1.0,1.0,1.0,1.0 +app_default_no_cutoff,127,3,663.32887,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,128,3,833.03858,0.0,0.0,0.0,0.0 +app_default_no_cutoff,129,3,788.74925,1.0,1.0,1.0,1.0 +app_default_no_cutoff,130,3,764.01121,1.0,1.0,1.0,1.0 +app_default_no_cutoff,132,3,720.27221,0.0,0.0,0.0,0.0 +app_default_no_cutoff,133,3,844.40858,0.55315,0.4,1.0,1.0 +app_default_no_cutoff,137,3,712.26187,1.0,1.0,1.0,1.0 +app_default_no_cutoff,141,3,609.59375,0.91972,1.0,1.0,1.0 +app_default_no_cutoff,142,3,883.28954,1.0,1.0,1.0,1.0 +app_default_no_cutoff,143,3,876.76633,1.0,1.0,1.0,1.0 +app_default_no_cutoff,146,3,813.74846,1.0,1.0,1.0,1.0 +app_default_no_cutoff,148,3,630.77446,1.0,1.0,1.0,1.0 +app_default_no_cutoff,163,3,553.61554,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,171,3,421.16096,1.0,1.0,1.0,1.0 +app_default_no_cutoff,179,3,677.65204,0.63668,0.5,1.0,1.0 +app_default_no_cutoff,180,3,821.77542,1.0,1.0,1.0,1.0 +app_default_no_cutoff,183,3,473.97758,1.0,1.0,1.0,1.0 +app_default_no_cutoff,185,3,827.49554,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,198,3,735.36333,0.0,0.0,0.0,0.0 +app_default_no_cutoff,208,3,819.74742,1.0,1.0,1.0,1.0 +app_default_no_cutoff,212,3,366.42642,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,213,3,642.5605,0.0,0.0,0.0,0.0 +app_default_no_cutoff,216,3,628.339,1.0,1.0,1.0,1.0 +app_default_no_cutoff,217,3,779.23912,1.0,1.0,1.0,1.0 +app_default_no_cutoff,218,3,726.63008,1.0,1.0,1.0,1.0 +app_default_no_cutoff,219,3,676.727,1.0,1.0,1.0,1.0 +app_default_no_cutoff,230,3,522.2575,1.0,1.0,1.0,1.0 +app_default_no_cutoff,232,3,644.99129,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,233,3,449.55479,1.0,1.0,1.0,1.0 +app_default_no_cutoff,236,3,486.35683,1.0,1.0,1.0,1.0 +app_default_no_cutoff,237,3,617.50362,0.0,0.0,0.0,0.0 +app_default_no_cutoff,238,3,450.85212,0.0,0.0,0.0,0.0 +app_default_no_cutoff,239,3,382.16458,0.0,0.0,0.0,0.0 +app_default_no_cutoff,248,3,759.68442,1.0,1.0,1.0,1.0 +app_default_no_cutoff,249,3,778.08517,1.0,1.0,1.0,1.0 +app_default_no_cutoff,261,3,693.85367,0.38685,0.5,0.5,1.0 +app_default_no_cutoff,268,3,585.82213,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,269,3,420.78167,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,274,3,628.289,1.0,1.0,1.0,1.0 +app_default_no_cutoff,275,3,715.91733,0.70392,0.66667,1.0,1.0 +app_default_no_cutoff,279,3,647.65783,1.0,1.0,1.0,1.0 +app_default_no_cutoff,294,3,488.05483,1.0,1.0,1.0,1.0 +app_default_no_cutoff,295,3,410.90025,1.0,1.0,1.0,1.0 +app_default_no_cutoff,298,3,755.0885,1.0,1.0,1.0,1.0 +app_default_no_cutoff,300,3,475.79942,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,303,3,844.97075,0.0,0.0,0.0,0.0 +app_default_no_cutoff,312,3,703.48704,0.0,0.0,0.0,0.0 +app_default_no_cutoff,314,3,781.53742,0.0,0.0,0.0,0.0 +app_default_no_cutoff,324,3,783.20637,0.0,0.0,0.0,0.0 +app_default_no_cutoff,327,3,669.73608,1.0,1.0,1.0,1.0 +app_default_no_cutoff,338,3,687.22712,1.0,1.0,1.0,1.0 +app_default_no_cutoff,343,3,753.23279,0.30657,0.5,0.33333,1.0 +app_default_no_cutoff,350,3,812.93037,1.0,1.0,1.0,1.0 +app_default_no_cutoff,354,3,393.66679,1.0,1.0,1.0,1.0 +app_default_no_cutoff,362,3,712.79929,1.0,1.0,1.0,1.0 +app_default_no_cutoff,380,3,812.11308,1.0,1.0,1.0,1.0 +app_default_no_cutoff,384,3,741.72521,0.0,0.0,0.0,0.0 +app_default_no_cutoff,385,3,729.51367,0.38685,0.5,0.5,1.0 +app_default_no_cutoff,386,3,741.16237,1.0,1.0,1.0,1.0 +app_default_no_cutoff,388,3,723.62971,0.0,0.0,0.0,0.0 +app_default_no_cutoff,399,3,608.96863,1.0,1.0,1.0,1.0 +app_default_no_cutoff,410,3,573.52733,1.0,1.0,1.0,1.0 +app_default_no_cutoff,411,3,608.22308,1.0,1.0,1.0,1.0 +app_default_no_cutoff,415,3,657.12125,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,421,3,825.48979,0.0,0.0,0.0,0.0 +app_default_no_cutoff,431,3,760.43146,0.0,0.0,0.0,0.0 +app_default_no_cutoff,436,3,704.06025,1.0,1.0,1.0,1.0 +app_default_no_cutoff,437,3,627.27438,0.0,0.0,0.0,0.0 +app_default_no_cutoff,439,3,657.72192,1.0,1.0,1.0,1.0 +app_default_no_cutoff,440,3,754.03492,1.0,1.0,1.0,1.0 +app_default_no_cutoff,443,3,739.89904,1.0,1.0,1.0,1.0 +app_default_no_cutoff,452,3,842.93992,0.38685,0.5,0.5,1.0 +app_default_no_cutoff,475,3,756.40862,0.0,0.0,0.0,0.0 +app_default_no_cutoff,478,3,475.33937,1.0,1.0,1.0,1.0 +app_default_no_cutoff,491,3,796.72454,1.0,1.0,1.0,1.0 +app_default_no_cutoff,501,3,803.41604,1.0,1.0,1.0,1.0 +app_default_no_cutoff,502,3,881.20704,0.0,0.0,0.0,0.0 +app_default_no_cutoff,507,3,710.99842,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,508,3,799.07063,0.0,0.0,0.0,0.0 +app_default_no_cutoff,513,3,785.90558,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,514,3,713.62433,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,516,3,595.19229,1.0,1.0,1.0,1.0 +app_default_no_cutoff,517,3,753.55558,0.0,0.0,0.0,0.0 +app_default_no_cutoff,521,3,649.74367,1.0,1.0,1.0,1.0 +app_default_no_cutoff,525,3,664.29058,1.0,1.0,1.0,1.0 +app_default_no_cutoff,527,3,749.91617,0.0,0.0,0.0,0.0 +app_default_no_cutoff,528,3,713.51437,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,532,3,729.57408,1.0,1.0,1.0,1.0 +app_default_no_cutoff,533,3,737.90833,1.0,1.0,1.0,1.0 +app_default_no_cutoff,535,3,780.07583,0.0,0.0,0.0,0.0 +app_default_no_cutoff,536,3,841.69508,1.0,1.0,1.0,1.0 +app_default_no_cutoff,539,3,736.43388,1.0,1.0,1.0,1.0 +app_default_no_cutoff,540,3,716.07954,0.30657,0.5,0.33333,1.0 +app_default_no_cutoff,544,3,789.29946,0.0,0.0,0.0,0.0 +app_default_no_cutoff,549,3,801.79079,1.0,1.0,1.0,1.0 +app_default_no_cutoff,551,3,760.76937,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,552,3,463.89662,1.0,1.0,1.0,1.0 +app_default_no_cutoff,554,3,710.46658,0.0,0.0,0.0,0.0 +app_default_no_cutoff,560,3,811.787,0.0,0.0,0.0,0.0 +app_default_no_cutoff,569,3,776.96996,1.0,1.0,1.0,1.0 +app_default_no_cutoff,575,3,508.68629,1.0,1.0,1.0,1.0 +app_default_no_cutoff,577,3,871.44679,0.0,0.0,0.0,0.0 +app_default_no_cutoff,578,3,850.70029,1.0,1.0,1.0,1.0 +app_default_no_cutoff,587,3,864.16346,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,589,3,724.6055,1.0,1.0,1.0,1.0 +app_default_no_cutoff,593,3,725.38388,1.0,1.0,1.0,1.0 +app_default_no_cutoff,597,3,555.33638,0.46928,0.33333,1.0,1.0 +app_default_no_cutoff,598,3,406.03933,0.0,0.0,0.0,0.0 +app_default_no_cutoff,613,3,795.15275,1.0,1.0,1.0,1.0 +app_default_no_cutoff,619,3,842.09471,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,623,3,739.87208,0.0,0.0,0.0,0.0 +app_default_no_cutoff,628,3,634.23717,1.0,1.0,1.0,1.0 +app_default_no_cutoff,636,3,628.27208,1.0,1.0,1.0,1.0 +app_default_no_cutoff,637,3,803.40154,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,641,3,785.40604,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,644,3,790.66788,1.0,1.0,1.0,1.0 +app_default_no_cutoff,649,3,691.09725,1.0,1.0,1.0,1.0 +app_default_no_cutoff,659,3,902.92975,0.0,0.0,0.0,0.0 +app_default_no_cutoff,660,3,961.62492,0.0,0.0,0.0,0.0 +app_default_no_cutoff,674,3,660.64142,1.0,1.0,1.0,1.0 +app_default_no_cutoff,684,3,588.61929,0.0,0.0,0.0,0.0 +app_default_no_cutoff,690,3,974.89687,0.0,0.0,0.0,0.0 +app_default_no_cutoff,691,3,834.34333,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,692,3,751.73863,1.0,1.0,1.0,1.0 +app_default_no_cutoff,693,3,772.89633,1.0,1.0,1.0,1.0 +app_default_no_cutoff,700,3,758.96058,0.0,0.0,0.0,0.0 +app_default_no_cutoff,702,3,686.37812,0.0,0.0,0.0,0.0 +app_default_no_cutoff,715,3,961.74862,0.0,0.0,0.0,0.0 +app_default_no_cutoff,716,3,866.02088,0.0,0.0,0.0,0.0 +app_default_no_cutoff,718,3,712.898,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,721,3,832.45088,1.0,1.0,1.0,1.0 +app_default_no_cutoff,723,3,382.23083,1.0,1.0,1.0,1.0 +app_default_no_cutoff,727,3,760.32683,0.0,0.0,0.0,0.0 +app_default_no_cutoff,728,3,815.40933,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,729,3,861.59708,1.0,1.0,1.0,1.0 +app_default_no_cutoff,742,3,831.49579,1.0,1.0,1.0,1.0 +app_default_no_cutoff,743,3,891.16487,1.0,1.0,1.0,1.0 +app_default_no_cutoff,744,3,362.34192,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,756,3,384.91354,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,759,3,550.86354,1.0,1.0,1.0,1.0 +app_default_no_cutoff,768,3,854.83963,0.0,0.0,0.0,0.0 +app_default_no_cutoff,770,3,795.78088,1.0,1.0,1.0,1.0 +app_default_no_cutoff,775,3,370.84467,0.0,0.0,0.0,0.0 +app_default_no_cutoff,781,3,430.85746,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,783,3,694.89233,0.0,0.0,0.0,0.0 +app_default_no_cutoff,784,3,676.50912,1.0,1.0,1.0,1.0 +app_default_no_cutoff,785,3,775.69396,0.0,0.0,0.0,0.0 +app_default_no_cutoff,793,3,708.82167,1.0,1.0,1.0,1.0 +app_default_no_cutoff,800,3,753.83342,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,805,3,603.31342,1.0,1.0,1.0,1.0 +app_default_no_cutoff,808,3,812.69638,1.0,1.0,1.0,1.0 +app_default_no_cutoff,811,3,916.23838,1.0,1.0,1.0,1.0 +app_default_no_cutoff,814,3,895.30783,1.0,1.0,1.0,1.0 +app_default_no_cutoff,820,3,722.1385,0.0,0.0,0.0,0.0 +app_default_no_cutoff,821,3,740.82937,0.0,0.0,0.0,0.0 +app_default_no_cutoff,823,3,667.67704,1.0,1.0,1.0,1.0 +app_default_no_cutoff,830,3,800.73967,0.0,0.0,0.0,0.0 +app_default_no_cutoff,831,3,754.00071,0.0,0.0,0.0,0.0 +app_default_no_cutoff,832,3,711.96458,1.0,1.0,1.0,1.0 +app_default_no_cutoff,834,3,470.33875,0.0,0.0,0.0,0.0 +app_default_no_cutoff,837,3,724.16454,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,839,3,486.98312,1.0,1.0,1.0,1.0 +app_default_no_cutoff,845,3,664.13504,1.0,1.0,1.0,1.0 +app_default_no_cutoff,847,3,706.14483,1.0,1.0,1.0,1.0 +app_default_no_cutoff,852,3,665.2295,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,859,3,721.1875,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,870,3,783.36808,0.0,0.0,0.0,0.0 +app_default_no_cutoff,873,3,844.80108,0.0,0.0,0.0,0.0 +app_default_no_cutoff,879,3,804.32796,1.0,1.0,1.0,1.0 +app_default_no_cutoff,880,3,820.48567,1.0,1.0,1.0,1.0 +app_default_no_cutoff,882,3,815.09033,1.0,1.0,1.0,1.0 +app_default_no_cutoff,887,3,725.01063,0.0,0.0,0.0,0.0 +app_default_no_cutoff,903,3,615.31979,1.0,1.0,1.0,1.0 +app_default_no_cutoff,904,3,772.69162,1.0,1.0,1.0,1.0 +app_default_no_cutoff,907,3,678.56929,1.0,1.0,1.0,1.0 +app_default_no_cutoff,911,3,786.8385,1.0,1.0,1.0,1.0 +app_default_no_cutoff,913,3,756.83762,0.0,0.0,0.0,0.0 +app_default_no_cutoff,914,3,807.11779,0.0,0.0,0.0,0.0 +app_default_no_cutoff,921,3,815.18554,1.0,1.0,1.0,1.0 +app_default_no_cutoff,922,3,765.94454,1.0,1.0,1.0,1.0 +app_default_no_cutoff,936,3,719.99787,1.0,1.0,1.0,1.0 +app_default_no_cutoff,956,3,769.75879,1.0,1.0,1.0,1.0 +app_default_no_cutoff,957,3,379.05475,1.0,1.0,1.0,1.0 +app_default_no_cutoff,960,3,709.40667,1.0,1.0,1.0,1.0 +app_default_no_cutoff,967,3,803.00083,0.91972,1.0,1.0,1.0 +app_default_no_cutoff,971,3,551.02796,0.63668,0.5,1.0,1.0 +app_default_no_cutoff,975,3,877.05171,0.0,0.0,0.0,0.0 +app_default_no_cutoff,982,3,806.42371,1.0,1.0,1.0,1.0 +app_default_no_cutoff,985,3,700.09471,1.0,1.0,1.0,1.0 +app_default_no_cutoff,993,3,589.50221,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1012,3,622.13388,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1014,3,876.101,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1019,3,809.9965,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1020,3,643.01183,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1021,3,676.70475,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1024,3,715.94192,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1029,3,755.02288,0.76536,0.66667,1.0,1.0 +app_default_no_cutoff,1041,3,377.01996,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1049,3,826.58375,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1062,3,439.72475,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1086,3,777.75542,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1088,3,746.82704,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1089,3,464.30604,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1099,3,567.6315,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1100,3,650.35642,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1104,3,839.80133,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1107,3,611.66246,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1110,3,822.96579,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1121,3,769.34587,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1130,3,703.67417,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1132,3,701.08258,0.61315,0.5,1.0,1.0 +app_default_no_cutoff,1137,3,763.30221,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1140,3,879.28417,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1144,3,630.29975,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1146,3,771.1235,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1150,3,724.93892,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1163,3,723.325,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1175,3,946.64704,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1179,3,889.13108,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1180,3,843.23008,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1185,3,793.17921,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1187,3,806.93442,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1191,3,936.67546,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1194,3,435.16858,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1196,3,377.95004,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1197,3,385.79596,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1199,3,803.55433,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1200,3,1012.43442,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1202,3,702.75692,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1204,3,726.74371,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1207,3,871.60254,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1213,3,740.74508,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1216,3,836.70067,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1221,3,821.02833,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1225,3,946.86187,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1226,3,920.02779,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1232,3,716.60475,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1241,3,777.74058,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1245,3,852.18592,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1259,3,847.64029,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1262,3,361.49367,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1266,3,606.58425,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,1270,3,718.89217,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1271,3,678.53012,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1272,3,880.09467,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,1273,3,806.63829,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1274,3,671.98729,0.76536,0.66667,1.0,1.0 +app_default_no_cutoff,1278,3,877.54242,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1279,3,821.34671,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1280,3,791.3235,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1281,3,833.45533,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1282,3,837.19567,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1290,3,767.49033,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1292,3,794.89312,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1298,3,765.08062,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1303,3,845.09425,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1316,3,529.14708,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1319,3,774.79812,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1320,3,707.5315,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1332,3,802.56983,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1335,3,586.4785,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1336,3,797.955,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1337,3,801.18225,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1339,3,834.23279,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1344,3,876.31996,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1352,3,757.33579,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1359,3,712.37275,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1362,3,948.14708,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1363,3,857.16658,0.5,1.0,0.33333,1.0 +app_default_no_cutoff,1368,3,643.13992,0.0,0.0,0.0,0.0 +app_default_no_cutoff,1370,3,766.12354,0.63093,1.0,0.5,1.0 +app_default_no_cutoff,1379,3,663.87804,0.39038,0.25,1.0,1.0 +app_default_no_cutoff,1382,3,789.15275,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1385,3,818.2315,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1389,3,956.65925,1.0,1.0,1.0,1.0 +app_default_no_cutoff,1395,3,740.55204,1.0,1.0,1.0,1.0 diff --git a/evaluation/results.json b/evaluation/results.json index 635f8f3..f3b48c6 100644 --- a/evaluation/results.json +++ b/evaluation/results.json @@ -37,47 +37,47 @@ "mrr@10": 0.6047, "hit@3": 0.6833, "no_passage_share": 0.0, - "ms_per_question_median": 9.5 + "ms_per_question_median": 9.8 }, "bm25": { - "ndcg@10": 0.5597, - "recall@10": 0.6862, - "mrr@10": 0.5242, - "hit@3": 0.59, + "ndcg@10": 0.6519, + "recall@10": 0.774, + "mrr@10": 0.6186, + "hit@3": 0.6933, "no_passage_share": 0.0, - "ms_per_question_median": 10.5 + "ms_per_question_median": 10.0 }, "dense+rerank": { - "ndcg@10": 0.6909, + "ndcg@10": 0.6906, "recall@10": 0.8102, - "mrr@10": 0.6619, + "mrr@10": 0.6616, "hit@3": 0.71, "no_passage_share": 0.0, - "ms_per_question_median": 1225.1 + "ms_per_question_median": 1247.3 }, "hybrid+rerank": { - "ndcg@10": 0.6935, - "recall@10": 0.8172, - "mrr@10": 0.6622, - "hit@3": 0.72, + "ndcg@10": 0.6962, + "recall@10": 0.8306, + "mrr@10": 0.6623, + "hit@3": 0.7233, "no_passage_share": 0.0, - "ms_per_question_median": 1871.6 + "ms_per_question_median": 1860.2 }, "app_default": { - "ndcg@10": 0.6029, - "recall@10": 0.6411, - "mrr@10": 0.6044, - "hit@3": 0.6667, - "no_passage_share": 0.1233, - "ms_per_question_median": 678.7 + "ndcg@10": 0.6054, + "recall@10": 0.6433, + "mrr@10": 0.6078, + "hit@3": 0.67, + "no_passage_share": 0.12, + "ms_per_question_median": 692.6 }, "app_default_no_cutoff": { - "ndcg@10": 0.6392, - "recall@10": 0.6861, - "mrr@10": 0.6389, - "hit@3": 0.7133, + "ndcg@10": 0.6467, + "recall@10": 0.6983, + "mrr@10": 0.6456, + "hit@3": 0.7267, "no_passage_share": 0.0, - "ms_per_question_median": 655.9 + "ms_per_question_median": 741.0 } } } diff --git a/src/rag_engine.py b/src/rag_engine.py index 2fdfbb0..a87edfa 100644 --- a/src/rag_engine.py +++ b/src/rag_engine.py @@ -35,6 +35,15 @@ BM25_AVAILABLE = False +def bm25_tokens(text: str) -> List[str]: + """Lower-case words for BM25, without punctuation, so that "cancer." matches "cancer". + + Splitting on spaces alone kept punctuation on words; on the SciFact benchmark this + tokenizer raised BM25 nDCG@10 from 0.560 to 0.652 (see evaluation/). + """ + return re.findall(r"\w+", text.lower()) + + class RAGEngine: """Enhanced RAG Engine with PDF processing and two-stage retrieval.""" @@ -1192,9 +1201,7 @@ def _bm25_search( metas = all_data.get("metadatas", []) or [{}] * len(corpus) if not corpus: return [], [] - tokenized_corpus = [ - doc.lower().split() for doc in corpus - ] + tokenized_corpus = [bm25_tokens(doc) for doc in corpus] bm25 = BM25Okapi(tokenized_corpus) # Store in cache self._bm25_cache = { @@ -1215,9 +1222,7 @@ def _bm25_search( ): bm25 = self._bm25_cache["bm25"] else: - tokenized_corpus = [ - doc.lower().split() for doc in corpus - ] + tokenized_corpus = [bm25_tokens(doc) for doc in corpus] bm25 = BM25Okapi(tokenized_corpus) self._bm25_cache = { "corpus_size": corpus_size, @@ -1226,7 +1231,7 @@ def _bm25_search( "bm25": bm25, } - query_tokens = query.lower().split() + query_tokens = bm25_tokens(query) scores = bm25.get_scores(query_tokens) top_indices = np.argsort(scores)[::-1][:top_n] diff --git a/tests/test_bm25_tokens.py b/tests/test_bm25_tokens.py new file mode 100644 index 0000000..489af14 --- /dev/null +++ b/tests/test_bm25_tokens.py @@ -0,0 +1,24 @@ +"""Checks for the words BM25 sees. Needs the full requirements (the engine imports its models).""" + +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("chromadb") +pytest.importorskip("sentence_transformers") +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from rag_engine import bm25_tokens # noqa: E402 + + +def test_punctuation_is_not_part_of_a_word(): + assert bm25_tokens("Cancer. (cancer), CANCER!") == ["cancer", "cancer", "cancer"] + + +def test_turkish_letters_stay_in_words(): + assert bm25_tokens("Şirketin geliri: 5,2 milyon") == ["şirketin", "geliri", "5", "2", "milyon"] + + +def test_empty_text_gives_no_words(): + assert bm25_tokens(" ... ") == [] From 9d38b29c555643b45c719c9b6a675f49c27fa22e Mon Sep 17 00:00:00 2001 From: YsK-dev Date: Sat, 14 Nov 2026 14:34:28 +0300 Subject: [PATCH 3/4] README: measured search quality, with what the numbers do and do not show --- README.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0c65936..f49a2c4 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,57 @@ repository. - **Basic safety checks:** common prompt-injection phrases in questions are rejected. - **Conversations:** follow-up questions see the last 20 messages of the conversation. +## How well the search works + +Measured on [SciFact](https://huggingface.co/datasets/BeIR/scifact), a public benchmark from +BEIR: 5,183 scientific abstracts and 300 test questions, each with the abstracts that answer it. +Every abstract is stored as one passage in the app's own search engine (`src/rag_engine.py`); +reading files and splitting them is not part of this test. Laptop CPU (Apple M1), no GPU. + +| Search set-up | nDCG@10 | Recall@10 | Hit@3 | Time per question | +|---|---|---|---|---| +| Vector search only | 0.645 | 78.3% | 68.3% | 10 ms | +| BM25 keyword search only | 0.652 | 77.4% | 69.3% | 10 ms | +| Vector search, then reranker | 0.691 | 81.0% | 71.0% | 1.2 s | +| **Vector + BM25, then reranker (what the app does)** | **0.696** | **83.1%** | **72.3%** | **1.9 s** | + +- **nDCG@10** is the usual score for this benchmark: 1.0 means the right abstracts are at the + top of the first 10 results, 0 means none of them is there. **Recall@10** is the share of + right abstracts found in the first 10. **Hit@3** is the share of questions with a right + abstract in the first 3, the number the app passes to the language model by default. + Time is the median per question. +- **The reranker is what helps.** It lifts nDCG@10 by 0.05 over vector search alone (95% + interval +0.02 to +0.08, paired over the 300 questions), at about 1.2 s per question on this CPU. +- **BM25 adds little on top of the reranker here:** +0.006 nDCG@10 (95% interval −0.009 to + +0.022), for about 0.6 s more per question. It may matter more for documents full of names, + codes or numbers; SciFact does not test that. +- **A fix found by this test.** BM25 used to split text on spaces only, so "cancer." and + "cancer" were different words. Splitting into words raised BM25 alone from 0.560 to 0.652 + (95% interval +0.07 to +0.12). The full pipeline barely moved (0.694 to 0.696). + +**The app's default call** keeps at most 3 passages and drops passages with a reranker confidence +below 0.3; when nothing is left, the app says it cannot answer. + +| Default call | Hit@3 | Questions with no passage | +|---|---|---| +| With the 0.3 cut-off (as the app runs) | 67.0% | 12.0% (36 of 300) | +| Without the cut-off | 72.7% | 0% | + +For 26 of those 36 questions, the passages that were dropped were all wrong, so the app was +right not to answer from them. For the other 10, a right passage was dropped. Every SciFact +question has an answer in the abstracts, so this test shows what the cut-off costs, not how often +it saves the app from answering a question the documents cannot answer. + +Reproduce (downloads the pinned benchmark and models, about 25 minutes on a laptop CPU): + +```bash +pip install -r requirements.txt +python -m evaluation.retrieval_eval --dataset scifact +``` + +Results: `evaluation/results.json`, one row per question and set-up: +`evaluation/per_question_scifact.csv`. + ## Run it You need Python 3.10 or newer and [Ollama](https://ollama.com/). @@ -74,7 +125,8 @@ curl -X POST http://localhost:8000/api/chat \ ## Tests The unit tests check every endpoint in a light test mode that does not load any model, so they -run in about a second. +run in about a second. The tests of the search metrics run there too; the BM25 word test runs +when the full requirements are installed. ```bash pip install -r requirements-dev.txt From 0b3cb56ca550f3219361b26a66b3c8a18936e799 Mon Sep 17 00:00:00 2001 From: YsK-dev Date: Sat, 14 Nov 2026 14:46:08 +0300 Subject: [PATCH 4/4] Reproducible intervals, independent check of the BM25 fix, README corrections - evaluation/compare.py: paired bootstrap interval between two set-ups or two runs. - evaluation/bm25_tokens_check.py: BM25 with the old and new word splitting on another BEIR dataset. On FiQA (not used to choose the fix): nDCG@10 0.159 -> 0.217, Hit@3 23.0% -> 30.9%. - requirements-eval.txt: pandas and pyarrow for the measurement. - README: the 10-result pipeline row is labelled as such; the default call's cost of the cut-off in full (17 claims, 156 with fewer than 3 passages); claims, not questions; whole abstracts, not the app's passages; history is the last 6 messages; tables split every 50 rows; FAISS needs faiss-cpu; first start downloads two models. --- README.md | 85 +++++++++++++++++-------------- evaluation/bm25_tokens_check.json | 19 +++++++ evaluation/bm25_tokens_check.py | 64 +++++++++++++++++++++++ evaluation/compare.py | 60 ++++++++++++++++++++++ requirements-eval.txt | 4 ++ 5 files changed, 195 insertions(+), 37 deletions(-) create mode 100644 evaluation/bm25_tokens_check.json create mode 100644 evaluation/bm25_tokens_check.py create mode 100644 evaluation/compare.py create mode 100644 requirements-eval.txt diff --git a/README.md b/README.md index f49a2c4..226bd96 100644 --- a/README.md +++ b/README.md @@ -12,71 +12,79 @@ repository. - **Reads many file types:** PDF (with tables), Word, Excel, CSV, PowerPoint, HTML, Markdown and plain text. - **Splits documents into passages:** text is cut into pieces of about 500 characters with 100 - characters of overlap, and small neighbouring pieces are joined up to 1,000 characters. Tables, - code and formulas are kept whole. A passage that repeats, such as a page header, is kept at - most twice. + characters of overlap, and small neighbouring pieces are joined up to 1,000 characters. Tables + are kept as tables, 50 rows per passage with the header repeated. A + passage that repeats, such as a page header, is kept at most twice. - **Finds the right passages in two steps:** 1. Candidates come from two searches: by meaning (vector search with `all-MiniLM-L6-v2` in - ChromaDB, or FAISS) and by keywords (BM25). + ChromaDB; FAISS also works after `pip install faiss-cpu`) and by keywords (BM25). 2. A cross-encoder (`ms-marco-MiniLM-L-6-v2`) reads each question–passage pair and puts the candidates in order. Passages it is not confident about (below 0.3) are dropped. If nothing is left, the app says it cannot answer instead of guessing. - **Answers with a local model:** Qwen3 1.7B (fast) or Qwen3 4B Thinking, through Ollama. The answer streams word by word and lists the passages it used. - **Basic safety checks:** common prompt-injection phrases in questions are rejected. -- **Conversations:** follow-up questions see the last 20 messages of the conversation. +- **Conversations:** follow-up questions see the last 6 messages (3 questions and answers). ## How well the search works Measured on [SciFact](https://huggingface.co/datasets/BeIR/scifact), a public benchmark from -BEIR: 5,183 scientific abstracts and 300 test questions, each with the abstracts that answer it. -Every abstract is stored as one passage in the app's own search engine (`src/rag_engine.py`); -reading files and splitting them is not part of this test. Laptop CPU (Apple M1), no GPU. +BEIR: 5,183 scientific abstracts and 300 test claims (short scientific statements), each with the +abstracts that support or contradict it. Every abstract is stored whole as one passage in the +app's own search engine (`src/rag_engine.py`); reading files and splitting them into the app's +smaller passages is not part of this test. Laptop CPU (Apple M1), no GPU. -| Search set-up | nDCG@10 | Recall@10 | Hit@3 | Time per question | +| Search set-up (10 results) | nDCG@10 | Recall@10 | Hit@3 | Time per claim | |---|---|---|---|---| | Vector search only | 0.645 | 78.3% | 68.3% | 10 ms | | BM25 keyword search only | 0.652 | 77.4% | 69.3% | 10 ms | -| Vector search, then reranker | 0.691 | 81.0% | 71.0% | 1.2 s | -| **Vector + BM25, then reranker (what the app does)** | **0.696** | **83.1%** | **72.3%** | **1.9 s** | +| Vector search, then reranker (30 candidates) | 0.691 | 81.0% | 71.0% | 1.2 s | +| Vector + BM25, then reranker (30 + 30 candidates, no cut-off) | 0.696 | 83.1% | 72.3% | 1.9 s | - **nDCG@10** is the usual score for this benchmark: 1.0 means the right abstracts are at the top of the first 10 results, 0 means none of them is there. **Recall@10** is the share of - right abstracts found in the first 10. **Hit@3** is the share of questions with a right - abstract in the first 3, the number the app passes to the language model by default. - Time is the median per question. -- **The reranker is what helps.** It lifts nDCG@10 by 0.05 over vector search alone (95% - interval +0.02 to +0.08, paired over the 300 questions), at about 1.2 s per question on this CPU. + right abstracts found in the first 10. **Hit@3** is the share of claims with a right abstract + in the first 3. Time is the median per claim. +- **The reranker is what helps.** It lifts nDCG@10 by 0.046 over vector search alone (95% + interval +0.016 to +0.075, paired over the 300 claims), at about 1.2 s per claim on this CPU. - **BM25 adds little on top of the reranker here:** +0.006 nDCG@10 (95% interval −0.009 to - +0.022), for about 0.6 s more per question. It may matter more for documents full of names, - codes or numbers; SciFact does not test that. -- **A fix found by this test.** BM25 used to split text on spaces only, so "cancer." and - "cancer" were different words. Splitting into words raised BM25 alone from 0.560 to 0.652 - (95% interval +0.07 to +0.12). The full pipeline barely moved (0.694 to 0.696). - -**The app's default call** keeps at most 3 passages and drops passages with a reranker confidence -below 0.3; when nothing is left, the app says it cannot answer. - -| Default call | Hit@3 | Questions with no passage | + +0.022), for about 0.6 s more per claim at 10 results. It may help more on other kinds of + documents; SciFact does not test that. +- **A bug found by this test.** BM25 used to split text on spaces only, so "cancer." and "cancer" + were different words. Splitting into words raised BM25 alone from 0.560 to 0.652 (95% interval + +0.066 to +0.119); the full pipeline barely moved (0.694 to 0.696). The numbers before the fix + are in commit `4350ee6`. The new splitting was chosen while looking at SciFact, so it was also + checked on a dataset that played no part in that choice: on + [FiQA](https://huggingface.co/datasets/BeIR/fiqa) (57,600 financial forum posts, 648 test + questions), BM25 alone went from 0.159 to 0.217 nDCG@10 and Hit@3 from 23.0% to 30.9% + (`python -m evaluation.bm25_tokens_check --dataset fiqa`, results in + `evaluation/bm25_tokens_check.json`). + +**The app's default call** asks for 3 passages (9 vector + 9 BM25 candidates) and drops passages +with a reranker confidence below 0.3; when nothing is left, the app says it cannot answer. It takes +about 0.7 s per claim. + +| Default call | Hit@3 | Claims with no passage | |---|---|---| | With the 0.3 cut-off (as the app runs) | 67.0% | 12.0% (36 of 300) | | Without the cut-off | 72.7% | 0% | -For 26 of those 36 questions, the passages that were dropped were all wrong, so the app was -right not to answer from them. For the other 10, a right passage was dropped. Every SciFact -question has an answer in the abstracts, so this test shows what the cut-off costs, not how often -it saves the app from answering a question the documents cannot answer. +The cut-off costs 17 claims a right passage: 10 that get no passage at all and 7 whose list was +cut to 1 or 2 passages. In all, 156 of the 300 claims get fewer than 3 passages. For the other +26 claims with no passage, the 3 passages the app would have used were all wrong, so not +answering was the better outcome. Every SciFact claim has an answer in the abstracts, so this test +shows what the cut-off costs, not how often it saves the app from answering a question the +documents cannot answer. Whole abstracts are longer than the app's passages, so the share of +dropped passages and the timings may differ on your own documents. Reproduce (downloads the pinned benchmark and models, about 25 minutes on a laptop CPU): ```bash -pip install -r requirements.txt -python -m evaluation.retrieval_eval --dataset scifact +pip install -r requirements-eval.txt +python -m evaluation.retrieval_eval --dataset scifact # results.json, per_question_scifact.csv +python -m evaluation.compare dense+rerank dense # paired difference with 95% interval ``` -Results: `evaluation/results.json`, one row per question and set-up: -`evaluation/per_question_scifact.csv`. - ## Run it You need Python 3.10 or newer and [Ollama](https://ollama.com/). @@ -96,6 +104,8 @@ pip install -r requirements.txt uvicorn main:app --reload # http://localhost:8000, API page at /docs ``` +The first start downloads the embedding model and the reranker from Hugging Face (about 180 MB in all). + Try it: ```bash @@ -124,8 +134,9 @@ curl -X POST http://localhost:8000/api/chat \ ## Tests -The unit tests check every endpoint in a light test mode that does not load any model, so they -run in about a second. The tests of the search metrics run there too; the BM25 word test runs +The unit tests check the main endpoints (chat, streaming, upload, documents, history, feedback) +and input checks in a light test mode that does not load any model, so they run in about a +second. The tests of the search metrics run there too; the BM25 word test runs when the full requirements are installed. ```bash diff --git a/evaluation/bm25_tokens_check.json b/evaluation/bm25_tokens_check.json new file mode 100644 index 0000000..5e60efb --- /dev/null +++ b/evaluation/bm25_tokens_check.json @@ -0,0 +1,19 @@ +{ + "fiqa": { + "dataset": "fiqa", + "documents": 57600, + "test_questions": 648, + "tokenizers": { + "split_on_spaces": { + "ndcg@10": 0.1591, + "recall@10": 0.2037, + "hit@3": 0.2299 + }, + "words": { + "ndcg@10": 0.2167, + "recall@10": 0.278, + "hit@3": 0.3086 + } + } + } +} diff --git a/evaluation/bm25_tokens_check.py b/evaluation/bm25_tokens_check.py new file mode 100644 index 0000000..83b4ce4 --- /dev/null +++ b/evaluation/bm25_tokens_check.py @@ -0,0 +1,64 @@ +"""BM25 alone with the old word splitting (on spaces) and the new one (words without punctuation). + +The new splitting was chosen while looking at SciFact, so this script also runs on other BEIR +datasets as an independent check. It uses rank_bm25 directly, as the app does, and needs no models. + + python -m evaluation.bm25_tokens_check --dataset fiqa +""" + +import argparse +import json +import re +from pathlib import Path + +import numpy as np +from rank_bm25 import BM25Okapi + +from evaluation.metrics import hit_at_k, ndcg_at_k, recall_at_k +from evaluation.retrieval_eval import DATASETS, load + +TOKENIZERS = { + "split_on_spaces": lambda text: text.lower().split(), + "words": lambda text: re.findall(r"\w+", text.lower()), # the app's bm25_tokens() +} +OUT = Path(__file__).resolve().parent / "bm25_tokens_check.json" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--dataset", choices=sorted(DATASETS), default="fiqa") + args = parser.parse_args() + + corpus, questions, relevant = load(args.dataset) + titles = corpus["title"].fillna("").astype(str) + bodies = corpus["text"].fillna("").astype(str) + texts = [(t + "\n" + b).strip() for t, b in zip(titles, bodies)] + ids = corpus["_id"].astype(str).tolist() + keep = [i for i, t in enumerate(texts) if t] + texts, ids = [texts[i] for i in keep], [ids[i] for i in keep] + print(f"{args.dataset}: {len(texts)} documents, {len(questions)} test questions", flush=True) + + result = {"dataset": args.dataset, "documents": len(texts), "test_questions": len(questions), "tokenizers": {}} + for name, tokenize in TOKENIZERS.items(): + bm25 = BM25Okapi([tokenize(t) for t in texts]) + ndcg, recall, hit = [], [], [] + for qid, q in questions.items(): + s = bm25.get_scores(tokenize(q)) + top = np.argsort(s)[::-1][:10] + ranked = [ids[i] for i in top if s[i] > 0] + ndcg.append(ndcg_at_k(ranked, relevant[qid])) + recall.append(recall_at_k(ranked, relevant[qid])) + hit.append(hit_at_k(ranked, relevant[qid], 3)) + result["tokenizers"][name] = {"ndcg@10": round(float(np.mean(ndcg)), 4), + "recall@10": round(float(np.mean(recall)), 4), + "hit@3": round(float(np.mean(hit)), 4)} + print(f" {name}: {result['tokenizers'][name]}", flush=True) + + saved = json.loads(OUT.read_text()) if OUT.exists() else {} + saved[args.dataset] = result + OUT.write_text(json.dumps(saved, indent=2) + "\n") + print(f"Saved {OUT}") + + +if __name__ == "__main__": + main() diff --git a/evaluation/compare.py b/evaluation/compare.py new file mode 100644 index 0000000..002a0e2 --- /dev/null +++ b/evaluation/compare.py @@ -0,0 +1,60 @@ +"""Compare two search set-ups question by question, with a 95% bootstrap interval. + +The difference A - B is computed per question, then the questions are resampled with +replacement (5,000 times, seed 0) to get the interval. + + python -m evaluation.compare dense+rerank dense + python -m evaluation.compare hybrid+rerank dense+rerank + python -m evaluation.compare app_default app_default_no_cutoff --metric hit3 + + # The same set-up in two runs, for example before and after a code change: + git show 4350ee6:evaluation/per_question_scifact.csv > before.csv + python -m evaluation.compare bm25 bm25 --csv-b before.csv +""" + +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +DEFAULT_CSV = Path(__file__).resolve().parent / "per_question_scifact.csv" + + +def paired_bootstrap(diff: np.ndarray, resamples: int = 5000, seed: int = 0) -> tuple[float, float, float]: + """Mean of the per-question differences and its 95% interval.""" + rng = np.random.default_rng(seed) + idx = rng.integers(0, len(diff), size=(resamples, len(diff))) + means = diff[idx].mean(axis=1) + return float(diff.mean()), float(np.percentile(means, 2.5)), float(np.percentile(means, 97.5)) + + +def scores(csv: Path, setup: str, metric: str) -> pd.Series: + table = pd.read_csv(csv, dtype={"query_id": str}) + rows = table[table["setup"] == setup] + if rows.empty: + raise SystemExit(f"No set-up '{setup}' in {csv}. Set-ups: {sorted(table['setup'].unique())}") + return rows.set_index("query_id")[metric] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("a") + parser.add_argument("b") + parser.add_argument("--metric", default="ndcg10", choices=["ndcg10", "recall10", "mrr10", "hit3"]) + parser.add_argument("--csv", type=Path, default=DEFAULT_CSV, help="per-question results for A (and B)") + parser.add_argument("--csv-b", type=Path, help="per-question results for B, if from another run") + args = parser.parse_args() + + a = scores(args.csv, args.a, args.metric) + b = scores(args.csv_b or args.csv, args.b, args.metric) + if set(a.index) != set(b.index): + raise SystemExit("The two result sets do not cover the same questions.") + diff = (a - b.reindex(a.index)).to_numpy(dtype=float) + mean, low, high = paired_bootstrap(diff) + print(f"{args.metric}: {args.a} {a.mean():.4f}, {args.b} {b.mean():.4f}, " + f"difference {mean:+.4f} (95% interval {low:+.4f} to {high:+.4f}, {len(diff)} questions)") + + +if __name__ == "__main__": + main() diff --git a/requirements-eval.txt b/requirements-eval.txt new file mode 100644 index 0000000..0f09ef3 --- /dev/null +++ b/requirements-eval.txt @@ -0,0 +1,4 @@ +# For the search measurement in evaluation/ (on top of the app's own requirements). +-r requirements.txt +pandas +pyarrow