{ "provenance": { "geoql_env": "0.1.0", "engine": "wiktorn/overpass-api:v0.7.62.11@sha256:5f643f2aa500333f19458fe8be14457215d6e0b9f891ca5c5ed452b1f8882c47", "worlds": { "ad-andorra": "dab3167c7c583dc9efb06a80479b08bbfa322ce56ba18ba0f8ca47e47ca35c0a", "lu-luxembourg-city": "d8f3e1dabb4d9d5489500c35432eb393089c32222cfff8cf50706852f0696a80", "mc-monaco": "2c0debd7676c0d0d80ec534251cb004a6797441d14512cb83d01fe1ad81363bf", "mv-maldives": "22f8606c845ef2eaff54d5b550b486595cdb900ffeafffb0cc62b2897197f837" }, "tagdocs": "sha512-z2nQgHLNwCjebDxlKBaERjg02UmcJyemAtsXExJFlXNNc4K9l7EdtfkGs064Y0IgzUltmUxM/FeUBRyhb4kv6w==", "tasks": "0741a2bbbe847920eb5f20125d108298c8ade1c36a73d9e9fda9309d76b557ba", "gold": "172a3c16405238a3cad35ddb83983a1444af7de66412829cd7faa2875bf36703", "tag_counts": "94db7a7656e2c9458ffdf0c8f1e3e8410dd68fed0c702005a0f73ff388240e73", "prompt": "a6c16acc42ae863c666bab94f8dc4a6770cdeaed830fd495b7b192e7f29af961" }, "run": { "model": "openai-api/adaption/deepseek-v4-flash", "mode": "single_shot", "tools": "-" }, "summary": { "tasks": 67, "intents": 60, "epochs": 3, "accuracy": 0.29850746268656714, "ci95": [ 0.20707070707070707, 0.3939393939393939 ], "real_accuracy": 0.5572139303482587, "inflation": 0.2587064676616916, "inflation_ci95": [ 0.18181818181818182, 0.33838383838383834 ], "agreement": { "real": 0.5422885572139303, "w1": 0.47761194029850745, "w2": 0.47761194029850745, "w3": 0.29850746268656714 }, "epoch_accuracy": { "1": 0.373134328358209, "2": 0.26865671641791045, "3": 0.2537313432835821 }, "epoch_sd": 0.06505819318717423, "mean_f1": 0.5767243781094528, "valid": 0.8905472636815921, "categories": { "wrong_real": 70, "pass": 60, "wrong_witness": 49, "query_error": 11, "rejected": 7, "no_query": 2, "runtime_error": 2 }, "families": { "attribute_filters": { "tasks": 6, "accuracy": 0.2222222222222222 }, "containment": { "tasks": 6, "accuracy": 0.3333333333333333 }, "counting": { "tasks": 6, "accuracy": 0.1111111111111111 }, "element_types": { "tasks": 6, "accuracy": 0.38888888888888884 }, "judgment_calls": { "tasks": 6, "accuracy": 0.2222222222222222 }, "name_matching": { "tasks": 6, "accuracy": 0.16666666666666666 }, "proximity": { "tasks": 6, "accuracy": 0.5 }, "relations": { "tasks": 7, "accuracy": 0.14285714285714285 }, "set_composition": { "tasks": 6, "accuracy": 0.16666666666666666 }, "tag_grounding": { "tasks": 6, "accuracy": 0.27777777777777773 }, "viewport": { "tasks": 6, "accuracy": 0.7777777777777778 } }, "tag_frequency": { "common": { "tasks": 21, "accuracy": 0.30158730158730157 }, "uncommon": { "tasks": 24, "accuracy": 0.38888888888888884 }, "rare": { "tasks": 22, "accuracy": 0.19696969696969696 } }, "drafted_by": { "model": { "tasks": 67, "accuracy": 0.29850746268656714 } }, "pairs": { "pairs": 7, "consistency": 0.2857142857142857, "accuracy": 0.4523809523809524 }, "missing": [], "overpassnl": { "em": 0.014925373134328358, "chrf": 0.6365343446090134, "kvs": 0.6223837022252068, "trees": 0.46633470178246295, "oqs": 0.5750842495388944, "ex": 0.5074626865671642, "ex_soft": 0.5561983354966216, "real_first": 0.5074626865671642 }, "execution_agreement": { "both": 59, "execution_only": 43, "grade_only": 1, "neither": 98 } }, "records": [ { "task": "ad-01", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9406324339193626, "kvs": 0.0, "trees": 0.0, "oqs": 0.3135441446397875, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-01", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6027780276569459, "kvs": 0.36363636363636365, "trees": 0.625, "oqs": 0.5304714637644365, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-01", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7284287079031408, "kvs": 0.0, "trees": 0.0, "oqs": 0.24280956930104694, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-03", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5103013759685102, "kvs": 0.0, "trees": 0.0, "oqs": 0.17010045865617007, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-03", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5336753096186937, "kvs": 0.0, "trees": 0.0, "oqs": 0.1778917698728979, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-03", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5654658337584944, "kvs": 0.625, "trees": 0.38461538461538464, "oqs": 0.525027072791293, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-04", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7688252236664518, "kvs": 1.0, "trees": 0.42857142857142855, "oqs": 0.73246555074596, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-04", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6307256053375945, "kvs": 0.5, "trees": 0.5714285714285714, "oqs": 0.567384725588722, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-04", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6740855622332805, "kvs": 0.5, "trees": 0.2857142857142857, "oqs": 0.48659994931585543, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-05", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8589254395215422, "kvs": 1.0, "trees": 0.625, "oqs": 0.8279751465071806, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-05", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8636896637497093, "kvs": 1.0, "trees": 0.75, "oqs": 0.8712298879165697, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-05", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7099390290450162, "kvs": 0.75, "trees": 0.5, "oqs": 0.653313009681672, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-06", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.47433675328124986, "kvs": 0.14285714285714285, "trees": 0.7142857142857143, "oqs": 0.4438265368080357, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-06", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.8836450015420797, "kvs": 0.7, "trees": 0.7142857142857143, "oqs": 0.7659769052759313, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-06", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5847217890426891, "kvs": 0.14285714285714285, "trees": 0.5714285714285714, "oqs": 0.4330025011094678, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-07", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.49978271150427966, "kvs": 0.5, "trees": 0.375, "oqs": 0.45826090383475987, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-07", "epoch": 2, "category": "wrong_real", "f1": 0.9565, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6483280945775751, "kvs": 0.5, "trees": 0.375, "oqs": 0.5077760315258584, "ex": false, "ex_soft": 0.9166666666666666, "real_first": false } }, { "task": "ad-07", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.648086347345019, "kvs": 0.6666666666666666, "trees": 0.375, "oqs": 0.5632510046705619, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-08", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8027866456254904, "kvs": 0.6, "trees": 0.6, "oqs": 0.6675955485418301, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-08", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6446095548430312, "kvs": 1.0, "trees": 0.4, "oqs": 0.6815365182810104, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-08", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6739085882281888, "kvs": 1.0, "trees": 0.4, "oqs": 0.6913028627427296, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-09", "epoch": 1, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5620218623113149, "kvs": 0.0, "trees": 0.0, "oqs": 0.1873406207704383, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-09", "epoch": 2, "category": "no_query", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.0, "kvs": 0.0, "trees": 0.0, "oqs": 0.0, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-09", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.46746044825150557, "kvs": 0.23529411764705882, "trees": 0.23076923076923078, "oqs": 0.3111745988892651, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-10", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6644351841975915, "kvs": 0.5, "trees": 0.5555555555555556, "oqs": 0.5733302465843824, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-10", "epoch": 2, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6863719544988223, "kvs": 0.42857142857142855, "trees": 0.7222222222222222, "oqs": 0.612388535097491, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-10", "epoch": 3, "category": "wrong_real", "f1": 0.7143, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6950659604256391, "kvs": 0.7142857142857143, "trees": 0.5555555555555556, "oqs": 0.6549690767556363, "ex": false, "ex_soft": 0.5555555555555556, "real_first": false } }, { "task": "ad-11", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7499568256813862, "kvs": 1.0, "trees": 0.42857142857142855, "oqs": 0.7261760847509383, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-11", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.712385097943752, "kvs": 1.0, "trees": 0.2857142857142857, "oqs": 0.6660331278860125, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-11", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.8277168198629343, "kvs": 1.0, "trees": 0.5714285714285714, "oqs": 0.7997151304305019, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-12", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8036587674653537, "kvs": 1.0, "trees": 1.0, "oqs": 0.9345529224884513, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-12", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6818898323664665, "kvs": 1.0, "trees": 0.4, "oqs": 0.6939632774554888, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-12", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6571568425106197, "kvs": 1.0, "trees": 0.4, "oqs": 0.6857189475035398, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-13", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8555906839176423, "kvs": 1.0, "trees": 0.4, "oqs": 0.7518635613058807, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-13", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6145672888319947, "kvs": 1.0, "trees": 0.6, "oqs": 0.7381890962773316, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-13", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8555906839176423, "kvs": 1.0, "trees": 0.4, "oqs": 0.7518635613058807, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-14", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6916908938317984, "kvs": 0.7, "trees": 0.7692307692307693, "oqs": 0.7203072210208559, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-14", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5893638040788589, "kvs": 0.0, "trees": 0.0, "oqs": 0.19645460135961965, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-14", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5950687730134218, "kvs": 0.7, "trees": 0.5384615384615384, "oqs": 0.6111767704916534, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-15", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4861516703271872, "kvs": 0.6, "trees": 0.42857142857142855, "oqs": 0.504907699632872, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-15", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5599927695341697, "kvs": 0.6, "trees": 0.42857142857142855, "oqs": 0.5295213993685327, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-15", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7419268031742341, "kvs": 1.0, "trees": 0.42857142857142855, "oqs": 0.7234994105818875, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-16", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.7255504429038022, "kvs": 0.625, "trees": 0.6666666666666666, "oqs": 0.6724057031901562, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-16", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6759535396748639, "kvs": 0.625, "trees": 0.5, "oqs": 0.600317846558288, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-16", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7451795955769507, "kvs": 1.0, "trees": 0.5, "oqs": 0.7483931985256502, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-17", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5861358607037116, "kvs": 0.6, "trees": 0.5, "oqs": 0.5620452869012372, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-17", "epoch": 2, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.621925400534256, "kvs": 0.0, "trees": 0.0, "oqs": 0.20730846684475202, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "ad-17", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6437630168622784, "kvs": 0.8333333333333334, "trees": 0.5, "oqs": 0.6590321167318706, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "ad-18", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.453319499085873, "kvs": 0.5, "trees": 0.45454545454545453, "oqs": 0.4692883178771092, "ex": false, "ex_soft": 0.2962962962962963, "real_first": false } }, { "task": "ad-18", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.49046159287304597, "kvs": 0.5, "trees": 0.36363636363636365, "oqs": 0.4513659855031366, "ex": false, "ex_soft": 0.2962962962962963, "real_first": false } }, { "task": "ad-18", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6430364972465413, "kvs": 0.375, "trees": 0.6363636363636364, "oqs": 0.5514667112033925, "ex": false, "ex_soft": 0.2962962962962963, "real_first": false } }, { "task": "lu-02", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6768570339259954, "kvs": 1.0, "trees": 0.6, "oqs": 0.7589523446419985, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-02", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9146334583833994, "kvs": 1.0, "trees": 0.6, "oqs": 0.8382111527944666, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-02", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8069947360051514, "kvs": 1.0, "trees": 1.0, "oqs": 0.9356649120017172, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-03", "epoch": 1, "category": "rejected", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5276998195545239, "kvs": 0.5, "trees": 0.375, "oqs": 0.46756660651817467, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-03", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4948212688425423, "kvs": 0.3333333333333333, "trees": 0.375, "oqs": 0.4010515340586252, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-03", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5027181241247011, "kvs": 0.3333333333333333, "trees": 0.375, "oqs": 0.40368381915267815, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-04", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4005301852652616, "kvs": 0.16666666666666666, "trees": 0.375, "oqs": 0.31406561731064275, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-04", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.3745875728796816, "kvs": 0.16666666666666666, "trees": 0.375, "oqs": 0.3054180798487827, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-04", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.3889418606102405, "kvs": 0.16666666666666666, "trees": 0.375, "oqs": 0.3102028424256357, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-05", "epoch": 1, "category": "rejected", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6109016652004432, "kvs": 1.0, "trees": 0.75, "oqs": 0.7869672217334811, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-05", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5127511249820373, "kvs": 0.6666666666666666, "trees": 0.5, "oqs": 0.559805930549568, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-05", "epoch": 3, "category": "rejected", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6742702340470604, "kvs": 0.6666666666666666, "trees": 0.75, "oqs": 0.6969789669045756, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-06", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5925278807074672, "kvs": 1.0, "trees": 0.4, "oqs": 0.6641759602358225, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-06", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9394673351057425, "kvs": 1.0, "trees": 0.6, "oqs": 0.8464891117019141, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-06", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8248228839597508, "kvs": 1.0, "trees": 0.6, "oqs": 0.8082742946532503, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-07", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5730585207110385, "kvs": 0.6666666666666666, "trees": 0.42857142857142855, "oqs": 0.5560988719830445, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-07", "epoch": 2, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.45532815585540165, "kvs": 0.0, "trees": 0.0, "oqs": 0.15177605195180055, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-07", "epoch": 3, "category": "rejected", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5276877808018823, "kvs": 0.6666666666666666, "trees": 0.42857142857142855, "oqs": 0.5409752920133258, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-08", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.3836962775079494, "kvs": 1.0, "trees": 0.2, "oqs": 0.5278987591693164, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-08", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9032394731726606, "kvs": 1.0, "trees": 0.6, "oqs": 0.8344131577242203, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-08", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9115497660733758, "kvs": 1.0, "trees": 0.6, "oqs": 0.8371832553577919, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-09", "epoch": 1, "category": "rejected", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7504019112663283, "kvs": 1.0, "trees": 0.8571428571428571, "oqs": 0.8691815894697285, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-09", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5487746961860347, "kvs": 0.25, "trees": 0.5714285714285714, "oqs": 0.456734422538202, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-09", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4539111906711856, "kvs": 0.25, "trees": 0.5714285714285714, "oqs": 0.42511325403325234, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-10", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4767284084493787, "kvs": 0.3333333333333333, "trees": 0.46153846153846156, "oqs": 0.4238667344403912, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-10", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.8476309845309918, "kvs": 0.5, "trees": 0.8461538461538461, "oqs": 0.7312616102282794, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-10", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.8881999351259607, "kvs": 0.7272727272727273, "trees": 0.7692307692307693, "oqs": 0.7949011438764857, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-11", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4983601789678819, "kvs": 0.3, "trees": 0.2857142857142857, "oqs": 0.36135815489405587, "ex": false, "ex_soft": 0.75, "real_first": false } }, { "task": "lu-11", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5378345644692429, "kvs": 0.375, "trees": 0.2857142857142857, "oqs": 0.39951628339450956, "ex": false, "ex_soft": 0.75, "real_first": false } }, { "task": "lu-11", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6510044605188662, "kvs": 0.6, "trees": 0.2857142857142857, "oqs": 0.5122395820777172, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-12", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6367124782297169, "kvs": 0.6, "trees": 0.2857142857142857, "oqs": 0.5074755879813342, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-12", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.51351896621149, "kvs": 0.375, "trees": 0.2857142857142857, "oqs": 0.39141108397525853, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-12", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.7054807001347504, "kvs": 0.6, "trees": 0.5714285714285714, "oqs": 0.6256364238544406, "ex": false, "ex_soft": 0.75, "real_first": false } }, { "task": "lu-13", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9908768137695908, "kvs": 1.0, "trees": 1.0, "oqs": 0.996958937923197, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-13", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7869918215852111, "kvs": 1.0, "trees": 0.5384615384615384, "oqs": 0.7751511200155831, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-13", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.9493070547744755, "kvs": 1.0, "trees": 0.6923076923076923, "oqs": 0.8805382490273893, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-14", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6417103635331471, "kvs": 0.0, "trees": 0.0, "oqs": 0.21390345451104906, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-14", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7069412823973855, "kvs": 0.5, "trees": 0.6666666666666666, "oqs": 0.6245359830213507, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-14", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7869655358198145, "kvs": 0.0, "trees": 0.0, "oqs": 0.2623218452732715, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-15", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6582491641906895, "kvs": 0.75, "trees": 0.375, "oqs": 0.5944163880635632, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-15", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6296715774534195, "kvs": 0.75, "trees": 0.625, "oqs": 0.6682238591511398, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-15", "epoch": 3, "category": "rejected", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.620081526995695, "kvs": 1.0, "trees": 0.625, "oqs": 0.748360508998565, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-16", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.440032087772035, "kvs": 0.375, "trees": 0.375, "oqs": 0.3966773625906783, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-16", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6910085576356046, "kvs": 0.6666666666666666, "trees": 0.5, "oqs": 0.6192250747674238, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-16", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6754414456290593, "kvs": 0.6666666666666666, "trees": 0.5, "oqs": 0.6140360374319086, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-17", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5854386495633812, "kvs": 0.375, "trees": 0.2857142857142857, "oqs": 0.41538431175922225, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-17", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6505776837620523, "kvs": 0.5, "trees": 0.2857142857142857, "oqs": 0.47876398982544605, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-17", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "B", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.37803057302479165, "kvs": 0.2857142857142857, "trees": 0.42857142857142855, "oqs": 0.36410542910350197, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-18", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6022410846461692, "kvs": 0.7142857142857143, "trees": 0.6153846153846154, "oqs": 0.6439704714388329, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "lu-18", "epoch": 2, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5616714991340461, "kvs": 0.0, "trees": 0.0, "oqs": 0.18722383304468204, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "lu-18", "epoch": 3, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.504580782264326, "kvs": 0.0, "trees": 0.0, "oqs": 0.16819359408810866, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-01", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4522451375218472, "kvs": 0.5, "trees": 0.42857142857142855, "oqs": 0.4602721886977586, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-01", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7194850671608908, "kvs": 0.5, "trees": 0.5714285714285714, "oqs": 0.596971212863154, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-01", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5349689510596329, "kvs": 0.0, "trees": 0.0, "oqs": 0.1783229836865443, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-02", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7654343807718184, "kvs": 1.0, "trees": 0.5714285714285714, "oqs": 0.7789543174001299, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-02", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6141141919372585, "kvs": 0.5, "trees": 0.5714285714285714, "oqs": 0.5618475877886099, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-02", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6141141919372585, "kvs": 0.5, "trees": 0.5714285714285714, "oqs": 0.5618475877886099, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-04", "epoch": 1, "category": "runtime_error", "f1": 0.0, "interpretation": null, "real": true, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6265152688440825, "kvs": 0.5, "trees": 0.42857142857142855, "oqs": 0.518362232471837, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-04", "epoch": 2, "category": "wrong_real", "f1": 0.65, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6319523965850002, "kvs": 0.5454545454545454, "trees": 0.5714285714285714, "oqs": 0.582945171156039, "ex": false, "ex_soft": 0.48148148148148145, "real_first": false } }, { "task": "mc-04", "epoch": 3, "category": "runtime_error", "f1": 0.0, "interpretation": null, "real": true, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5251004546313971, "kvs": 0.2857142857142857, "trees": 0.5714285714285714, "oqs": 0.4607477705914181, "ex": false, "ex_soft": 0.5652173913043478, "real_first": false } }, { "task": "mc-05", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5168524119761364, "kvs": 0.6666666666666666, "trees": 0.5, "oqs": 0.5611730262142677, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-05", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5989886327051107, "kvs": 1.0, "trees": 0.375, "oqs": 0.6579962109017036, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-05", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.725203203425028, "kvs": 1.0, "trees": 0.375, "oqs": 0.7000677344750094, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-06", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.7715085986759733, "kvs": 1.0, "trees": 0.5714285714285714, "oqs": 0.7809790567015149, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-06", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6174647404170909, "kvs": 0.5, "trees": 0.2857142857142857, "oqs": 0.4677263420437921, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-06", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5478644283880475, "kvs": 0.5, "trees": 0.2857142857142857, "oqs": 0.444526238034111, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-07", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.844750982445449, "kvs": 1.0, "trees": 0.75, "oqs": 0.864916994148483, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-07", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.844750982445449, "kvs": 1.0, "trees": 0.75, "oqs": 0.864916994148483, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-07", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.823722056778655, "kvs": 1.0, "trees": 0.625, "oqs": 0.8162406855928849, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-08", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.42949939372918805, "kvs": 0.7272727272727273, "trees": 0.38461538461538464, "oqs": 0.5137958352057667, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-08", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.42720572918560057, "kvs": 0.4, "trees": 0.3076923076923077, "oqs": 0.3782993456259694, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-08", "epoch": 3, "category": "wrong_real", "f1": 0.642, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4935502325823582, "kvs": 1.0, "trees": 0.3076923076923077, "oqs": 0.6004141800915553, "ex": false, "ex_soft": 0.49056603773584906, "real_first": false } }, { "task": "mc-09", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6993341576480844, "kvs": 0.3333333333333333, "trees": 0.36363636363636365, "oqs": 0.4654346182059272, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-09", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8559241950408129, "kvs": 1.0, "trees": 0.8181818181818182, "oqs": 0.8913686710742104, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-09", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4098756289848168, "kvs": 0.3333333333333333, "trees": 0.36363636363636365, "oqs": 0.36894844198483795, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-10", "epoch": 1, "category": "wrong_real", "f1": 0.7742, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5531648511025825, "kvs": 0.3, "trees": 0.5, "oqs": 0.45105495036752746, "ex": false, "ex_soft": 0.631578947368421, "real_first": false } }, { "task": "mc-10", "epoch": 2, "category": "wrong_real", "f1": 0.8485, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6687513875559299, "kvs": 0.42857142857142855, "trees": 0.8333333333333334, "oqs": 0.6435520498202306, "ex": false, "ex_soft": 0.7368421052631579, "real_first": false } }, { "task": "mc-10", "epoch": 3, "category": "wrong_real", "f1": 0.7848, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6256460343527513, "kvs": 0.375, "trees": 0.5, "oqs": 0.5002153447842504, "ex": false, "ex_soft": 0.6526315789473685, "real_first": false } }, { "task": "mc-11", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8448831392734547, "kvs": 1.0, "trees": 0.8571428571428571, "oqs": 0.9006753321387707, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-11", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6545131908346951, "kvs": 0.5, "trees": 0.5714285714285714, "oqs": 0.5753139207544221, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-11", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5323484557220873, "kvs": 1.0, "trees": 0.42857142857142855, "oqs": 0.653639961431172, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-12", "epoch": 1, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5703730964736948, "kvs": 0.0, "trees": 0.0, "oqs": 0.1901243654912316, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-12", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7684250629175927, "kvs": 0.631578947368421, "trees": 0.7142857142857143, "oqs": 0.7047632415239095, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-12", "epoch": 3, "category": "wrong_real", "f1": 0.6667, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6869794348079751, "kvs": 0.5, "trees": 0.7142857142857143, "oqs": 0.6337550496978964, "ex": false, "ex_soft": 0.5, "real_first": false } }, { "task": "mc-13", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5730716797172525, "kvs": 1.0, "trees": 0.4, "oqs": 0.6576905599057508, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-13", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6541341218891208, "kvs": 1.0, "trees": 1.0, "oqs": 0.8847113739630403, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-13", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9399494995866878, "kvs": 1.0, "trees": 0.8, "oqs": 0.9133164998622293, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-14", "epoch": 1, "category": "no_query", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.0, "kvs": 0.0, "trees": 0.0, "oqs": 0.0, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-14", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5991331205725068, "kvs": 0.0, "trees": 0.0, "oqs": 0.1997110401908356, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-14", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6053835309101459, "kvs": 0.0, "trees": 0.0, "oqs": 0.20179451030338194, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-15", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5236006859524606, "kvs": 0.6, "trees": 0.42857142857142855, "oqs": 0.5173907048412963, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-15", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6456098562785506, "kvs": 1.0, "trees": 0.5714285714285714, "oqs": 0.7390128092357072, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-15", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4546788577090587, "kvs": 0.6, "trees": 0.42857142857142855, "oqs": 0.4944167620934958, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-16", "epoch": 1, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.3547286477160448, "kvs": 0.0, "trees": 0.0, "oqs": 0.11824288257201493, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-16", "epoch": 2, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4271224167241708, "kvs": 0.0, "trees": 0.0, "oqs": 0.14237413890805692, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-16", "epoch": 3, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.41708382507282216, "kvs": 0.0, "trees": 0.0, "oqs": 0.1390279416909407, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mc-17", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5721441955030349, "kvs": 1.0, "trees": 0.4, "oqs": 0.6573813985010116, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-17", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6341221696107738, "kvs": 1.0, "trees": 0.4, "oqs": 0.6780407232035913, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mc-17", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9552538286745692, "kvs": 1.0, "trees": 0.6, "oqs": 0.8517512762248565, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-01", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5640331320368788, "kvs": 1.0, "trees": 0.5714285714285714, "oqs": 0.7118205678218167, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-01", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.669105208129157, "kvs": 0.5, "trees": 0.42857142857142855, "oqs": 0.5325588789001953, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-01", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4405867968153854, "kvs": 0.5, "trees": 0.2857142857142857, "oqs": 0.4087670275098903, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-02", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8176125884320399, "kvs": 1.0, "trees": 0.75, "oqs": 0.85587086281068, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-02", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7085324225374645, "kvs": 0.5454545454545454, "trees": 0.5, "oqs": 0.5846623226640033, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-02", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.812392079610205, "kvs": 1.0, "trees": 0.625, "oqs": 0.812464026536735, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-04", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5949356798326588, "kvs": 1.0, "trees": 0.6, "oqs": 0.7316452266108863, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-04", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.5647467491679999, "kvs": 1.0, "trees": 0.6, "oqs": 0.7215822497226667, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-04", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.3418012977240905, "kvs": 1.0, "trees": 0.2, "oqs": 0.5139337659080302, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-05", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.46611875073214876, "kvs": 0.625, "trees": 0.6363636363636364, "oqs": 0.5758274623652616, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-05", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4911248546573279, "kvs": 0.2, "trees": 0.36363636363636365, "oqs": 0.35158707276456386, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-05", "epoch": 3, "category": "wrong_real", "f1": 0.8846, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4864913830831854, "kvs": 0.2, "trees": 0.5454545454545454, "oqs": 0.41064864284591024, "ex": false, "ex_soft": 0.7931034482758621, "real_first": false } }, { "task": "mv-07", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8199101239410514, "kvs": 1.0, "trees": 0.625, "oqs": 0.8149700413136838, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-07", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5185605230844297, "kvs": 0.6666666666666666, "trees": 0.375, "oqs": 0.5200757299170321, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-07", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5143130078480475, "kvs": 0.6666666666666666, "trees": 0.375, "oqs": 0.5186598915049047, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-08", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": true, "chrf": 1.0, "kvs": 1.0, "trees": 1.0, "oqs": 1.0, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-08", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6395879684591744, "kvs": 1.0, "trees": 0.4, "oqs": 0.6798626561530581, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-08", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": true, "chrf": 1.0, "kvs": 1.0, "trees": 1.0, "oqs": 1.0, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-09", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5175336733163668, "kvs": 0.36363636363636365, "trees": 0.5, "oqs": 0.4603900123175768, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-09", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8465106713611195, "kvs": 1.0, "trees": 0.75, "oqs": 0.8655035571203732, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-09", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.641445927333795, "kvs": 0.4444444444444444, "trees": 0.625, "oqs": 0.5702967905927464, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-10", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5273973705286431, "kvs": 0.4444444444444444, "trees": 0.4, "oqs": 0.4572806049910292, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-10", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.7460367349765394, "kvs": 1.0, "trees": 0.6, "oqs": 0.7820122449921798, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-10", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.733451604129099, "kvs": 0.5, "trees": 0.6666666666666666, "oqs": 0.633372756931922, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-11", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.761333854384396, "kvs": 1.0, "trees": 0.5714285714285714, "oqs": 0.7775874752709893, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-11", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.8310217913103836, "kvs": 1.0, "trees": 0.7142857142857143, "oqs": 0.8484358351986994, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-11", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6101571743741904, "kvs": 0.5, "trees": 0.42857142857142855, "oqs": 0.5129095343152063, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-12", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6497870974172453, "kvs": 0.4444444444444444, "trees": 0.5, "oqs": 0.5314105139538966, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-12", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6243720501731338, "kvs": 0.6666666666666666, "trees": 0.3, "oqs": 0.5303462389466002, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-12", "epoch": 3, "category": "query_error", "f1": 0.0, "interpretation": null, "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.626612776210824, "kvs": 0.0, "trees": 0.0, "oqs": 0.208870925403608, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-13", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.8563713065406802, "kvs": 1.0, "trees": 0.4, "oqs": 0.7521237688468934, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-13", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.7219301429962345, "kvs": 1.0, "trees": 1.0, "oqs": 0.9073100476654115, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-13", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.664167619869704, "kvs": 1.0, "trees": 0.4, "oqs": 0.6880558732899013, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-14", "epoch": 1, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.9620056856643956, "kvs": 1.0, "trees": 1.0, "oqs": 0.9873352285547985, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-14", "epoch": 2, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": false, "chrf": 0.6362070563842355, "kvs": 1.0, "trees": 0.2, "oqs": 0.6120690187947452, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-14", "epoch": 3, "category": "pass", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": true, "w2": true, "w3": true }, "overpassnl": { "em": true, "chrf": 1.0, "kvs": 1.0, "trees": 1.0, "oqs": 1.0, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-15", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6079093531343142, "kvs": 0.5, "trees": 0.42857142857142855, "oqs": 0.5121602605685809, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-15", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "A", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7582952989703576, "kvs": 1.0, "trees": 0.7142857142857143, "oqs": 0.8241936710853573, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-15", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.6412375849655833, "kvs": 0.5, "trees": 0.5714285714285714, "oqs": 0.5708887187980516, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-16", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5224273534382596, "kvs": 0.0, "trees": 0.0, "oqs": 0.17414245114608654, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-16", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5436778735381514, "kvs": 0.5, "trees": 0.42857142857142855, "oqs": 0.49074976736986, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-16", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7124593113679824, "kvs": 0.0, "trees": 0.0, "oqs": 0.2374864371226608, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-17", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7457796294279447, "kvs": 0.75, "trees": 0.6, "oqs": 0.6985932098093149, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-17", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7390970824373172, "kvs": 0.75, "trees": 0.6, "oqs": 0.6963656941457724, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-17", "epoch": 3, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.7597223693966656, "kvs": 0.75, "trees": 0.6, "oqs": 0.7032407897988886, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-18", "epoch": 1, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.38694967126267654, "kvs": 1.0, "trees": 0.3333333333333333, "oqs": 0.57342766819867, "ex": false, "ex_soft": 0.16666666666666666, "real_first": false } }, { "task": "mv-18", "epoch": 2, "category": "wrong_real", "f1": 0.0, "interpretation": "A", "real": false, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.3686995004645874, "kvs": 1.0, "trees": 0.3333333333333333, "oqs": 0.5673442779326402, "ex": false, "ex_soft": 0.16666666666666666, "real_first": false } }, { "task": "mv-18", "epoch": 3, "category": "rejected", "f1": 0.0, "interpretation": null, "real": true, "worlds": { "real": false, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.4000089425923101, "kvs": 1.0, "trees": 0.3333333333333333, "oqs": 0.5777807586418812, "ex": true, "ex_soft": 1.0, "real_first": true } }, { "task": "mv-19", "epoch": 1, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.3695793780728469, "kvs": 0.25, "trees": 0.5, "oqs": 0.3731931260242823, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-19", "epoch": 2, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.391690623821312, "kvs": 0.25, "trees": 0.5, "oqs": 0.3805635412737707, "ex": false, "ex_soft": 0.0, "real_first": false } }, { "task": "mv-19", "epoch": 3, "category": "wrong_witness", "f1": 1.0, "interpretation": "B", "real": true, "worlds": { "real": true, "w1": false, "w2": false, "w3": false }, "overpassnl": { "em": false, "chrf": 0.5416204402231563, "kvs": 0.5, "trees": 0.5, "oqs": 0.5138734800743854, "ex": false, "ex_soft": 0.0, "real_first": false } } ] }