|
|
--- |
|
|
tags: |
|
|
- mteb |
|
|
- sentence-transformers |
|
|
- transformers |
|
|
model-index: |
|
|
- name: multilingual-e5-large-instruct |
|
|
results: |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_counterfactual |
|
|
name: MTEB AmazonCounterfactualClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.23880597014924 |
|
|
- type: ap |
|
|
value: 39.07351965022687 |
|
|
- type: f1 |
|
|
value: 70.04836733862683 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_counterfactual |
|
|
name: MTEB AmazonCounterfactualClassification (de) |
|
|
config: de |
|
|
split: test |
|
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 66.71306209850107 |
|
|
- type: ap |
|
|
value: 79.01499914759529 |
|
|
- type: f1 |
|
|
value: 64.81951817560703 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_counterfactual |
|
|
name: MTEB AmazonCounterfactualClassification (en-ext) |
|
|
config: en-ext |
|
|
split: test |
|
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.85307346326837 |
|
|
- type: ap |
|
|
value: 22.447519885878737 |
|
|
- type: f1 |
|
|
value: 61.0162730745633 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_counterfactual |
|
|
name: MTEB AmazonCounterfactualClassification (ja) |
|
|
config: ja |
|
|
split: test |
|
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.04925053533191 |
|
|
- type: ap |
|
|
value: 23.44983217128922 |
|
|
- type: f1 |
|
|
value: 62.5723230907759 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_polarity |
|
|
name: MTEB AmazonPolarityClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.28742500000001 |
|
|
- type: ap |
|
|
value: 94.8449918887462 |
|
|
- type: f1 |
|
|
value: 96.28680923610432 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 56.716 |
|
|
- type: f1 |
|
|
value: 55.76510398266401 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (de) |
|
|
config: de |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 52.99999999999999 |
|
|
- type: f1 |
|
|
value: 52.00829994765178 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (es) |
|
|
config: es |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 48.806000000000004 |
|
|
- type: f1 |
|
|
value: 48.082345914983634 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 48.507999999999996 |
|
|
- type: f1 |
|
|
value: 47.68752844642045 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (ja) |
|
|
config: ja |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 47.709999999999994 |
|
|
- type: f1 |
|
|
value: 47.05870376637181 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_reviews_multi |
|
|
name: MTEB AmazonReviewsClassification (zh) |
|
|
config: zh |
|
|
split: test |
|
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 44.662000000000006 |
|
|
- type: f1 |
|
|
value: 43.42371965372771 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: arguana |
|
|
name: MTEB ArguAna |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 31.721 |
|
|
- type: map_at_10 |
|
|
value: 49.221 |
|
|
- type: map_at_100 |
|
|
value: 49.884 |
|
|
- type: map_at_1000 |
|
|
value: 49.888 |
|
|
- type: map_at_3 |
|
|
value: 44.31 |
|
|
- type: map_at_5 |
|
|
value: 47.276 |
|
|
- type: mrr_at_1 |
|
|
value: 32.432 |
|
|
- type: mrr_at_10 |
|
|
value: 49.5 |
|
|
- type: mrr_at_100 |
|
|
value: 50.163000000000004 |
|
|
- type: mrr_at_1000 |
|
|
value: 50.166 |
|
|
- type: mrr_at_3 |
|
|
value: 44.618 |
|
|
- type: mrr_at_5 |
|
|
value: 47.541 |
|
|
- type: ndcg_at_1 |
|
|
value: 31.721 |
|
|
- type: ndcg_at_10 |
|
|
value: 58.384 |
|
|
- type: ndcg_at_100 |
|
|
value: 61.111000000000004 |
|
|
- type: ndcg_at_1000 |
|
|
value: 61.187999999999995 |
|
|
- type: ndcg_at_3 |
|
|
value: 48.386 |
|
|
- type: ndcg_at_5 |
|
|
value: 53.708999999999996 |
|
|
- type: precision_at_1 |
|
|
value: 31.721 |
|
|
- type: precision_at_10 |
|
|
value: 8.741 |
|
|
- type: precision_at_100 |
|
|
value: 0.991 |
|
|
- type: precision_at_1000 |
|
|
value: 0.1 |
|
|
- type: precision_at_3 |
|
|
value: 20.057 |
|
|
- type: precision_at_5 |
|
|
value: 14.609 |
|
|
- type: recall_at_1 |
|
|
value: 31.721 |
|
|
- type: recall_at_10 |
|
|
value: 87.411 |
|
|
- type: recall_at_100 |
|
|
value: 99.075 |
|
|
- type: recall_at_1000 |
|
|
value: 99.644 |
|
|
- type: recall_at_3 |
|
|
value: 60.171 |
|
|
- type: recall_at_5 |
|
|
value: 73.044 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/arxiv-clustering-p2p |
|
|
name: MTEB ArxivClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 46.40419580759799 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/arxiv-clustering-s2s |
|
|
name: MTEB ArxivClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 40.48593255007969 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/askubuntudupquestions-reranking |
|
|
name: MTEB AskUbuntuDupQuestions |
|
|
config: default |
|
|
split: test |
|
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 63.889179122289995 |
|
|
- type: mrr |
|
|
value: 77.61146286769556 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/biosses-sts |
|
|
name: MTEB BIOSSES |
|
|
config: default |
|
|
split: test |
|
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 88.15075203727929 |
|
|
- type: cos_sim_spearman |
|
|
value: 86.9622224570873 |
|
|
- type: euclidean_pearson |
|
|
value: 86.70473853624121 |
|
|
- type: euclidean_spearman |
|
|
value: 86.9622224570873 |
|
|
- type: manhattan_pearson |
|
|
value: 86.21089380980065 |
|
|
- type: manhattan_spearman |
|
|
value: 86.75318154937008 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/bucc-bitext-mining |
|
|
name: MTEB BUCC (de-en) |
|
|
config: de-en |
|
|
split: test |
|
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 99.65553235908142 |
|
|
- type: f1 |
|
|
value: 99.60681976339595 |
|
|
- type: precision |
|
|
value: 99.58246346555325 |
|
|
- type: recall |
|
|
value: 99.65553235908142 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/bucc-bitext-mining |
|
|
name: MTEB BUCC (fr-en) |
|
|
config: fr-en |
|
|
split: test |
|
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 99.26260180497468 |
|
|
- type: f1 |
|
|
value: 99.14520507740848 |
|
|
- type: precision |
|
|
value: 99.08650671362535 |
|
|
- type: recall |
|
|
value: 99.26260180497468 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/bucc-bitext-mining |
|
|
name: MTEB BUCC (ru-en) |
|
|
config: ru-en |
|
|
split: test |
|
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 98.07412538967787 |
|
|
- type: f1 |
|
|
value: 97.86629719431936 |
|
|
- type: precision |
|
|
value: 97.76238309664012 |
|
|
- type: recall |
|
|
value: 98.07412538967787 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/bucc-bitext-mining |
|
|
name: MTEB BUCC (zh-en) |
|
|
config: zh-en |
|
|
split: test |
|
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 99.42074776197998 |
|
|
- type: f1 |
|
|
value: 99.38564156573635 |
|
|
- type: precision |
|
|
value: 99.36808846761454 |
|
|
- type: recall |
|
|
value: 99.42074776197998 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/banking77 |
|
|
name: MTEB Banking77Classification |
|
|
config: default |
|
|
split: test |
|
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 85.73376623376623 |
|
|
- type: f1 |
|
|
value: 85.68480707214599 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/biorxiv-clustering-p2p |
|
|
name: MTEB BiorxivClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 40.935218072113855 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/biorxiv-clustering-s2s |
|
|
name: MTEB BiorxivClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 36.276389017675264 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: BeIR/cqadupstack |
|
|
name: MTEB CQADupstackRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 27.764166666666668 |
|
|
- type: map_at_10 |
|
|
value: 37.298166666666674 |
|
|
- type: map_at_100 |
|
|
value: 38.530166666666666 |
|
|
- type: map_at_1000 |
|
|
value: 38.64416666666667 |
|
|
- type: map_at_3 |
|
|
value: 34.484833333333334 |
|
|
- type: map_at_5 |
|
|
value: 36.0385 |
|
|
- type: mrr_at_1 |
|
|
value: 32.93558333333333 |
|
|
- type: mrr_at_10 |
|
|
value: 41.589749999999995 |
|
|
- type: mrr_at_100 |
|
|
value: 42.425333333333334 |
|
|
- type: mrr_at_1000 |
|
|
value: 42.476333333333336 |
|
|
- type: mrr_at_3 |
|
|
value: 39.26825 |
|
|
- type: mrr_at_5 |
|
|
value: 40.567083333333336 |
|
|
- type: ndcg_at_1 |
|
|
value: 32.93558333333333 |
|
|
- type: ndcg_at_10 |
|
|
value: 42.706583333333334 |
|
|
- type: ndcg_at_100 |
|
|
value: 47.82483333333333 |
|
|
- type: ndcg_at_1000 |
|
|
value: 49.95733333333334 |
|
|
- type: ndcg_at_3 |
|
|
value: 38.064750000000004 |
|
|
- type: ndcg_at_5 |
|
|
value: 40.18158333333333 |
|
|
- type: precision_at_1 |
|
|
value: 32.93558333333333 |
|
|
- type: precision_at_10 |
|
|
value: 7.459833333333334 |
|
|
- type: precision_at_100 |
|
|
value: 1.1830833333333335 |
|
|
- type: precision_at_1000 |
|
|
value: 0.15608333333333332 |
|
|
- type: precision_at_3 |
|
|
value: 17.5235 |
|
|
- type: precision_at_5 |
|
|
value: 12.349833333333333 |
|
|
- type: recall_at_1 |
|
|
value: 27.764166666666668 |
|
|
- type: recall_at_10 |
|
|
value: 54.31775 |
|
|
- type: recall_at_100 |
|
|
value: 76.74350000000001 |
|
|
- type: recall_at_1000 |
|
|
value: 91.45208333333332 |
|
|
- type: recall_at_3 |
|
|
value: 41.23425 |
|
|
- type: recall_at_5 |
|
|
value: 46.73983333333334 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: climate-fever |
|
|
name: MTEB ClimateFEVER |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 12.969 |
|
|
- type: map_at_10 |
|
|
value: 21.584999999999997 |
|
|
- type: map_at_100 |
|
|
value: 23.3 |
|
|
- type: map_at_1000 |
|
|
value: 23.5 |
|
|
- type: map_at_3 |
|
|
value: 18.218999999999998 |
|
|
- type: map_at_5 |
|
|
value: 19.983 |
|
|
- type: mrr_at_1 |
|
|
value: 29.316 |
|
|
- type: mrr_at_10 |
|
|
value: 40.033 |
|
|
- type: mrr_at_100 |
|
|
value: 40.96 |
|
|
- type: mrr_at_1000 |
|
|
value: 41.001 |
|
|
- type: mrr_at_3 |
|
|
value: 37.123 |
|
|
- type: mrr_at_5 |
|
|
value: 38.757999999999996 |
|
|
- type: ndcg_at_1 |
|
|
value: 29.316 |
|
|
- type: ndcg_at_10 |
|
|
value: 29.858 |
|
|
- type: ndcg_at_100 |
|
|
value: 36.756 |
|
|
- type: ndcg_at_1000 |
|
|
value: 40.245999999999995 |
|
|
- type: ndcg_at_3 |
|
|
value: 24.822 |
|
|
- type: ndcg_at_5 |
|
|
value: 26.565 |
|
|
- type: precision_at_1 |
|
|
value: 29.316 |
|
|
- type: precision_at_10 |
|
|
value: 9.186 |
|
|
- type: precision_at_100 |
|
|
value: 1.6549999999999998 |
|
|
- type: precision_at_1000 |
|
|
value: 0.22999999999999998 |
|
|
- type: precision_at_3 |
|
|
value: 18.436 |
|
|
- type: precision_at_5 |
|
|
value: 13.876 |
|
|
- type: recall_at_1 |
|
|
value: 12.969 |
|
|
- type: recall_at_10 |
|
|
value: 35.142 |
|
|
- type: recall_at_100 |
|
|
value: 59.143 |
|
|
- type: recall_at_1000 |
|
|
value: 78.594 |
|
|
- type: recall_at_3 |
|
|
value: 22.604 |
|
|
- type: recall_at_5 |
|
|
value: 27.883000000000003 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: dbpedia-entity |
|
|
name: MTEB DBPedia |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 8.527999999999999 |
|
|
- type: map_at_10 |
|
|
value: 17.974999999999998 |
|
|
- type: map_at_100 |
|
|
value: 25.665 |
|
|
- type: map_at_1000 |
|
|
value: 27.406000000000002 |
|
|
- type: map_at_3 |
|
|
value: 13.017999999999999 |
|
|
- type: map_at_5 |
|
|
value: 15.137 |
|
|
- type: mrr_at_1 |
|
|
value: 62.5 |
|
|
- type: mrr_at_10 |
|
|
value: 71.891 |
|
|
- type: mrr_at_100 |
|
|
value: 72.294 |
|
|
- type: mrr_at_1000 |
|
|
value: 72.296 |
|
|
- type: mrr_at_3 |
|
|
value: 69.958 |
|
|
- type: mrr_at_5 |
|
|
value: 71.121 |
|
|
- type: ndcg_at_1 |
|
|
value: 50.875 |
|
|
- type: ndcg_at_10 |
|
|
value: 38.36 |
|
|
- type: ndcg_at_100 |
|
|
value: 44.235 |
|
|
- type: ndcg_at_1000 |
|
|
value: 52.154 |
|
|
- type: ndcg_at_3 |
|
|
value: 43.008 |
|
|
- type: ndcg_at_5 |
|
|
value: 40.083999999999996 |
|
|
- type: precision_at_1 |
|
|
value: 62.5 |
|
|
- type: precision_at_10 |
|
|
value: 30.0 |
|
|
- type: precision_at_100 |
|
|
value: 10.038 |
|
|
- type: precision_at_1000 |
|
|
value: 2.0869999999999997 |
|
|
- type: precision_at_3 |
|
|
value: 46.833000000000006 |
|
|
- type: precision_at_5 |
|
|
value: 38.800000000000004 |
|
|
- type: recall_at_1 |
|
|
value: 8.527999999999999 |
|
|
- type: recall_at_10 |
|
|
value: 23.828 |
|
|
- type: recall_at_100 |
|
|
value: 52.322 |
|
|
- type: recall_at_1000 |
|
|
value: 77.143 |
|
|
- type: recall_at_3 |
|
|
value: 14.136000000000001 |
|
|
- type: recall_at_5 |
|
|
value: 17.761 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/emotion |
|
|
name: MTEB EmotionClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 51.51 |
|
|
- type: f1 |
|
|
value: 47.632159862049896 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: fever |
|
|
name: MTEB FEVER |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 60.734 |
|
|
- type: map_at_10 |
|
|
value: 72.442 |
|
|
- type: map_at_100 |
|
|
value: 72.735 |
|
|
- type: map_at_1000 |
|
|
value: 72.75 |
|
|
- type: map_at_3 |
|
|
value: 70.41199999999999 |
|
|
- type: map_at_5 |
|
|
value: 71.80499999999999 |
|
|
- type: mrr_at_1 |
|
|
value: 65.212 |
|
|
- type: mrr_at_10 |
|
|
value: 76.613 |
|
|
- type: mrr_at_100 |
|
|
value: 76.79899999999999 |
|
|
- type: mrr_at_1000 |
|
|
value: 76.801 |
|
|
- type: mrr_at_3 |
|
|
value: 74.8 |
|
|
- type: mrr_at_5 |
|
|
value: 76.12400000000001 |
|
|
- type: ndcg_at_1 |
|
|
value: 65.212 |
|
|
- type: ndcg_at_10 |
|
|
value: 77.988 |
|
|
- type: ndcg_at_100 |
|
|
value: 79.167 |
|
|
- type: ndcg_at_1000 |
|
|
value: 79.452 |
|
|
- type: ndcg_at_3 |
|
|
value: 74.362 |
|
|
- type: ndcg_at_5 |
|
|
value: 76.666 |
|
|
- type: precision_at_1 |
|
|
value: 65.212 |
|
|
- type: precision_at_10 |
|
|
value: 10.003 |
|
|
- type: precision_at_100 |
|
|
value: 1.077 |
|
|
- type: precision_at_1000 |
|
|
value: 0.11199999999999999 |
|
|
- type: precision_at_3 |
|
|
value: 29.518 |
|
|
- type: precision_at_5 |
|
|
value: 19.016 |
|
|
- type: recall_at_1 |
|
|
value: 60.734 |
|
|
- type: recall_at_10 |
|
|
value: 90.824 |
|
|
- type: recall_at_100 |
|
|
value: 95.71600000000001 |
|
|
- type: recall_at_1000 |
|
|
value: 97.577 |
|
|
- type: recall_at_3 |
|
|
value: 81.243 |
|
|
- type: recall_at_5 |
|
|
value: 86.90299999999999 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: fiqa |
|
|
name: MTEB FiQA2018 |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 23.845 |
|
|
- type: map_at_10 |
|
|
value: 39.281 |
|
|
- type: map_at_100 |
|
|
value: 41.422 |
|
|
- type: map_at_1000 |
|
|
value: 41.593 |
|
|
- type: map_at_3 |
|
|
value: 34.467 |
|
|
- type: map_at_5 |
|
|
value: 37.017 |
|
|
- type: mrr_at_1 |
|
|
value: 47.531 |
|
|
- type: mrr_at_10 |
|
|
value: 56.204 |
|
|
- type: mrr_at_100 |
|
|
value: 56.928999999999995 |
|
|
- type: mrr_at_1000 |
|
|
value: 56.962999999999994 |
|
|
- type: mrr_at_3 |
|
|
value: 54.115 |
|
|
- type: mrr_at_5 |
|
|
value: 55.373000000000005 |
|
|
- type: ndcg_at_1 |
|
|
value: 47.531 |
|
|
- type: ndcg_at_10 |
|
|
value: 47.711999999999996 |
|
|
- type: ndcg_at_100 |
|
|
value: 54.510999999999996 |
|
|
- type: ndcg_at_1000 |
|
|
value: 57.103 |
|
|
- type: ndcg_at_3 |
|
|
value: 44.145 |
|
|
- type: ndcg_at_5 |
|
|
value: 45.032 |
|
|
- type: precision_at_1 |
|
|
value: 47.531 |
|
|
- type: precision_at_10 |
|
|
value: 13.194 |
|
|
- type: precision_at_100 |
|
|
value: 2.045 |
|
|
- type: precision_at_1000 |
|
|
value: 0.249 |
|
|
- type: precision_at_3 |
|
|
value: 29.424 |
|
|
- type: precision_at_5 |
|
|
value: 21.451 |
|
|
- type: recall_at_1 |
|
|
value: 23.845 |
|
|
- type: recall_at_10 |
|
|
value: 54.967 |
|
|
- type: recall_at_100 |
|
|
value: 79.11399999999999 |
|
|
- type: recall_at_1000 |
|
|
value: 94.56700000000001 |
|
|
- type: recall_at_3 |
|
|
value: 40.256 |
|
|
- type: recall_at_5 |
|
|
value: 46.215 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: hotpotqa |
|
|
name: MTEB HotpotQA |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 37.819 |
|
|
- type: map_at_10 |
|
|
value: 60.889 |
|
|
- type: map_at_100 |
|
|
value: 61.717999999999996 |
|
|
- type: map_at_1000 |
|
|
value: 61.778 |
|
|
- type: map_at_3 |
|
|
value: 57.254000000000005 |
|
|
- type: map_at_5 |
|
|
value: 59.541 |
|
|
- type: mrr_at_1 |
|
|
value: 75.638 |
|
|
- type: mrr_at_10 |
|
|
value: 82.173 |
|
|
- type: mrr_at_100 |
|
|
value: 82.362 |
|
|
- type: mrr_at_1000 |
|
|
value: 82.37 |
|
|
- type: mrr_at_3 |
|
|
value: 81.089 |
|
|
- type: mrr_at_5 |
|
|
value: 81.827 |
|
|
- type: ndcg_at_1 |
|
|
value: 75.638 |
|
|
- type: ndcg_at_10 |
|
|
value: 69.317 |
|
|
- type: ndcg_at_100 |
|
|
value: 72.221 |
|
|
- type: ndcg_at_1000 |
|
|
value: 73.382 |
|
|
- type: ndcg_at_3 |
|
|
value: 64.14 |
|
|
- type: ndcg_at_5 |
|
|
value: 67.07600000000001 |
|
|
- type: precision_at_1 |
|
|
value: 75.638 |
|
|
- type: precision_at_10 |
|
|
value: 14.704999999999998 |
|
|
- type: precision_at_100 |
|
|
value: 1.698 |
|
|
- type: precision_at_1000 |
|
|
value: 0.185 |
|
|
- type: precision_at_3 |
|
|
value: 41.394999999999996 |
|
|
- type: precision_at_5 |
|
|
value: 27.162999999999997 |
|
|
- type: recall_at_1 |
|
|
value: 37.819 |
|
|
- type: recall_at_10 |
|
|
value: 73.52499999999999 |
|
|
- type: recall_at_100 |
|
|
value: 84.875 |
|
|
- type: recall_at_1000 |
|
|
value: 92.559 |
|
|
- type: recall_at_3 |
|
|
value: 62.092999999999996 |
|
|
- type: recall_at_5 |
|
|
value: 67.907 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/imdb |
|
|
name: MTEB ImdbClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.60079999999999 |
|
|
- type: ap |
|
|
value: 92.67396345347356 |
|
|
- type: f1 |
|
|
value: 94.5988098167121 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: msmarco |
|
|
name: MTEB MSMARCO |
|
|
config: default |
|
|
split: dev |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 21.285 |
|
|
- type: map_at_10 |
|
|
value: 33.436 |
|
|
- type: map_at_100 |
|
|
value: 34.63 |
|
|
- type: map_at_1000 |
|
|
value: 34.681 |
|
|
- type: map_at_3 |
|
|
value: 29.412 |
|
|
- type: map_at_5 |
|
|
value: 31.715 |
|
|
- type: mrr_at_1 |
|
|
value: 21.848 |
|
|
- type: mrr_at_10 |
|
|
value: 33.979 |
|
|
- type: mrr_at_100 |
|
|
value: 35.118 |
|
|
- type: mrr_at_1000 |
|
|
value: 35.162 |
|
|
- type: mrr_at_3 |
|
|
value: 30.036 |
|
|
- type: mrr_at_5 |
|
|
value: 32.298 |
|
|
- type: ndcg_at_1 |
|
|
value: 21.862000000000002 |
|
|
- type: ndcg_at_10 |
|
|
value: 40.43 |
|
|
- type: ndcg_at_100 |
|
|
value: 46.17 |
|
|
- type: ndcg_at_1000 |
|
|
value: 47.412 |
|
|
- type: ndcg_at_3 |
|
|
value: 32.221 |
|
|
- type: ndcg_at_5 |
|
|
value: 36.332 |
|
|
- type: precision_at_1 |
|
|
value: 21.862000000000002 |
|
|
- type: precision_at_10 |
|
|
value: 6.491 |
|
|
- type: precision_at_100 |
|
|
value: 0.935 |
|
|
- type: precision_at_1000 |
|
|
value: 0.104 |
|
|
- type: precision_at_3 |
|
|
value: 13.744 |
|
|
- type: precision_at_5 |
|
|
value: 10.331999999999999 |
|
|
- type: recall_at_1 |
|
|
value: 21.285 |
|
|
- type: recall_at_10 |
|
|
value: 62.083 |
|
|
- type: recall_at_100 |
|
|
value: 88.576 |
|
|
- type: recall_at_1000 |
|
|
value: 98.006 |
|
|
- type: recall_at_3 |
|
|
value: 39.729 |
|
|
- type: recall_at_5 |
|
|
value: 49.608000000000004 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.92612859097127 |
|
|
- type: f1 |
|
|
value: 93.82370333372853 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (de) |
|
|
config: de |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 92.67681036911807 |
|
|
- type: f1 |
|
|
value: 92.14191382411472 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (es) |
|
|
config: es |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 92.26817878585723 |
|
|
- type: f1 |
|
|
value: 91.92824250337878 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 89.96554963983714 |
|
|
- type: f1 |
|
|
value: 90.02859329630792 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (hi) |
|
|
config: hi |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 90.02509860164935 |
|
|
- type: f1 |
|
|
value: 89.30665159182062 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_domain |
|
|
name: MTEB MTOPDomainClassification (th) |
|
|
config: th |
|
|
split: test |
|
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 87.55515370705244 |
|
|
- type: f1 |
|
|
value: 87.94449232331907 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 82.4623803009576 |
|
|
- type: f1 |
|
|
value: 66.06738378772725 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (de) |
|
|
config: de |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 79.3716539870386 |
|
|
- type: f1 |
|
|
value: 60.37614033396853 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (es) |
|
|
config: es |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 80.34022681787857 |
|
|
- type: f1 |
|
|
value: 58.302008026952 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.72095208268087 |
|
|
- type: f1 |
|
|
value: 59.64524724009049 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (hi) |
|
|
config: hi |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.87020437432773 |
|
|
- type: f1 |
|
|
value: 57.80202694670567 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/mtop_intent |
|
|
name: MTEB MTOPIntentClassification (th) |
|
|
config: th |
|
|
split: test |
|
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.73598553345387 |
|
|
- type: f1 |
|
|
value: 58.19628250675031 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (af) |
|
|
config: af |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 67.6630800268998 |
|
|
- type: f1 |
|
|
value: 65.00996668051691 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (am) |
|
|
config: am |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 60.7128446536651 |
|
|
- type: f1 |
|
|
value: 57.95860594874963 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ar) |
|
|
config: ar |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 63.61129791526563 |
|
|
- type: f1 |
|
|
value: 59.75328290206483 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (az) |
|
|
config: az |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.00134498991257 |
|
|
- type: f1 |
|
|
value: 67.0230483991802 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (bn) |
|
|
config: bn |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 68.54068594485541 |
|
|
- type: f1 |
|
|
value: 65.54604628946976 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (cy) |
|
|
config: cy |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 63.032952252858095 |
|
|
- type: f1 |
|
|
value: 58.715741857057104 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (da) |
|
|
config: da |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.80901143241427 |
|
|
- type: f1 |
|
|
value: 68.33963989243877 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (de) |
|
|
config: de |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.47141896435777 |
|
|
- type: f1 |
|
|
value: 69.56765020308262 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (el) |
|
|
config: el |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.2373907195696 |
|
|
- type: f1 |
|
|
value: 69.04529836036467 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.05783456624076 |
|
|
- type: f1 |
|
|
value: 74.69430584708174 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (es) |
|
|
config: es |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.82111634162744 |
|
|
- type: f1 |
|
|
value: 70.77228952803762 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (fa) |
|
|
config: fa |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.25353059852051 |
|
|
- type: f1 |
|
|
value: 71.05310103416411 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (fi) |
|
|
config: fi |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.28648285137861 |
|
|
- type: f1 |
|
|
value: 69.08020473732226 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.31540013449899 |
|
|
- type: f1 |
|
|
value: 70.9426355465791 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (he) |
|
|
config: he |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 70.2151983860121 |
|
|
- type: f1 |
|
|
value: 67.52541755908858 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (hi) |
|
|
config: hi |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.58372562205784 |
|
|
- type: f1 |
|
|
value: 69.49769064229827 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (hu) |
|
|
config: hu |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.9233355749832 |
|
|
- type: f1 |
|
|
value: 69.36311548259593 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (hy) |
|
|
config: hy |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 68.07330195023538 |
|
|
- type: f1 |
|
|
value: 64.99882022345572 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (id) |
|
|
config: id |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.62273032952253 |
|
|
- type: f1 |
|
|
value: 70.6394885471001 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (is) |
|
|
config: is |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 65.77000672494957 |
|
|
- type: f1 |
|
|
value: 62.9368944815065 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (it) |
|
|
config: it |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.453261600538 |
|
|
- type: f1 |
|
|
value: 70.85069934666681 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ja) |
|
|
config: ja |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.6906523201076 |
|
|
- type: f1 |
|
|
value: 72.03249740074217 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (jv) |
|
|
config: jv |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 63.03631472763953 |
|
|
- type: f1 |
|
|
value: 59.3165215571852 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ka) |
|
|
config: ka |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 58.913920645595155 |
|
|
- type: f1 |
|
|
value: 57.367337711611285 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (km) |
|
|
config: km |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 54.42837928715535 |
|
|
- type: f1 |
|
|
value: 52.60527294970906 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (kn) |
|
|
config: kn |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 66.33490248823135 |
|
|
- type: f1 |
|
|
value: 63.213340969404065 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ko) |
|
|
config: ko |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 70.58507061197041 |
|
|
- type: f1 |
|
|
value: 68.40256628040486 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (lv) |
|
|
config: lv |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.11230665770006 |
|
|
- type: f1 |
|
|
value: 66.44863577842305 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ml) |
|
|
config: ml |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.70073974445192 |
|
|
- type: f1 |
|
|
value: 67.21291337273702 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (mn) |
|
|
config: mn |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 66.43913920645595 |
|
|
- type: f1 |
|
|
value: 64.09838087422806 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ms) |
|
|
config: ms |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 70.80026899798251 |
|
|
- type: f1 |
|
|
value: 68.76986742962444 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (my) |
|
|
config: my |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 64.78816408876934 |
|
|
- type: f1 |
|
|
value: 62.18781873428972 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (nb) |
|
|
config: nb |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.6577000672495 |
|
|
- type: f1 |
|
|
value: 68.75171511133003 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (nl) |
|
|
config: nl |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.42501681237391 |
|
|
- type: f1 |
|
|
value: 71.18434963451544 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (pl) |
|
|
config: pl |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.64828513786146 |
|
|
- type: f1 |
|
|
value: 70.67741914007422 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (pt) |
|
|
config: pt |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.62811028917284 |
|
|
- type: f1 |
|
|
value: 71.36402039740959 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ro) |
|
|
config: ro |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.88634835238736 |
|
|
- type: f1 |
|
|
value: 69.23701923480677 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ru) |
|
|
config: ru |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.15938130464022 |
|
|
- type: f1 |
|
|
value: 71.87792218993388 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (sl) |
|
|
config: sl |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.96301277740416 |
|
|
- type: f1 |
|
|
value: 67.29584200202983 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (sq) |
|
|
config: sq |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.49562878278412 |
|
|
- type: f1 |
|
|
value: 66.91716685679431 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (sv) |
|
|
config: sv |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.6805648957633 |
|
|
- type: f1 |
|
|
value: 72.02723592594374 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (sw) |
|
|
config: sw |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 63.00605245460659 |
|
|
- type: f1 |
|
|
value: 60.16716669482932 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ta) |
|
|
config: ta |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 66.90988567585742 |
|
|
- type: f1 |
|
|
value: 63.99405488777784 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (te) |
|
|
config: te |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 67.62273032952253 |
|
|
- type: f1 |
|
|
value: 65.17213906909481 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (th) |
|
|
config: th |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.50907868190988 |
|
|
- type: f1 |
|
|
value: 69.15165697194853 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (tl) |
|
|
config: tl |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.30733019502352 |
|
|
- type: f1 |
|
|
value: 66.69024007380474 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (tr) |
|
|
config: tr |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.24277067921989 |
|
|
- type: f1 |
|
|
value: 68.80515408492947 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (ur) |
|
|
config: ur |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 67.49831876260929 |
|
|
- type: f1 |
|
|
value: 64.83778567111116 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (vi) |
|
|
config: vi |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.28782784129119 |
|
|
- type: f1 |
|
|
value: 69.3294186700733 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
|
config: zh-CN |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.315400134499 |
|
|
- type: f1 |
|
|
value: 71.22674385243207 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_intent |
|
|
name: MTEB MassiveIntentClassification (zh-TW) |
|
|
config: zh-TW |
|
|
split: test |
|
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.37794216543377 |
|
|
- type: f1 |
|
|
value: 68.96962492838232 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (af) |
|
|
config: af |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.33557498318764 |
|
|
- type: f1 |
|
|
value: 72.28949738478356 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (am) |
|
|
config: am |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 65.84398117014123 |
|
|
- type: f1 |
|
|
value: 64.71026362091463 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ar) |
|
|
config: ar |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.76462676529925 |
|
|
- type: f1 |
|
|
value: 69.8229667407667 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (az) |
|
|
config: az |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.02420981842636 |
|
|
- type: f1 |
|
|
value: 71.76576384895898 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (bn) |
|
|
config: bn |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.7572293207801 |
|
|
- type: f1 |
|
|
value: 72.76840765295256 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (cy) |
|
|
config: cy |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 68.02286482851379 |
|
|
- type: f1 |
|
|
value: 66.17237947327872 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (da) |
|
|
config: da |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.60928043039678 |
|
|
- type: f1 |
|
|
value: 77.27094731234773 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (de) |
|
|
config: de |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.68325487558843 |
|
|
- type: f1 |
|
|
value: 77.97530399082261 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (el) |
|
|
config: el |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.13315400134498 |
|
|
- type: f1 |
|
|
value: 75.97558584796424 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 80.47410894418292 |
|
|
- type: f1 |
|
|
value: 80.52244841473792 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (es) |
|
|
config: es |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.9670477471419 |
|
|
- type: f1 |
|
|
value: 77.37318805793146 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (fa) |
|
|
config: fa |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 78.09683927370544 |
|
|
- type: f1 |
|
|
value: 77.69773737430847 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (fi) |
|
|
config: fi |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.20847343644922 |
|
|
- type: f1 |
|
|
value: 75.17071738727348 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (fr) |
|
|
config: fr |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.07464694014796 |
|
|
- type: f1 |
|
|
value: 77.16136207698571 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (he) |
|
|
config: he |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.53396099529255 |
|
|
- type: f1 |
|
|
value: 73.58296404484122 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (hi) |
|
|
config: hi |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.75319435104237 |
|
|
- type: f1 |
|
|
value: 75.24674707850833 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (hu) |
|
|
config: hu |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.0948217888366 |
|
|
- type: f1 |
|
|
value: 76.47559490205028 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (hy) |
|
|
config: hy |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.07599193006052 |
|
|
- type: f1 |
|
|
value: 70.76028043093511 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (id) |
|
|
config: id |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.10490921318089 |
|
|
- type: f1 |
|
|
value: 77.01215275283272 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (is) |
|
|
config: is |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.25756556825824 |
|
|
- type: f1 |
|
|
value: 70.20605314648762 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (it) |
|
|
config: it |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.08137188971082 |
|
|
- type: f1 |
|
|
value: 77.3899269057439 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ja) |
|
|
config: ja |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 79.35440484196369 |
|
|
- type: f1 |
|
|
value: 79.58964690002772 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (jv) |
|
|
config: jv |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 68.42299932750504 |
|
|
- type: f1 |
|
|
value: 68.07844356925413 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ka) |
|
|
config: ka |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 66.15669132481507 |
|
|
- type: f1 |
|
|
value: 65.89383352608513 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (km) |
|
|
config: km |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 60.11432414256894 |
|
|
- type: f1 |
|
|
value: 57.69910594559806 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (kn) |
|
|
config: kn |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.24747814391392 |
|
|
- type: f1 |
|
|
value: 70.42455553830918 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ko) |
|
|
config: ko |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.46267652992603 |
|
|
- type: f1 |
|
|
value: 76.8854559308316 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (lv) |
|
|
config: lv |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 73.24815063887021 |
|
|
- type: f1 |
|
|
value: 72.77805034658074 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ml) |
|
|
config: ml |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.11566913248151 |
|
|
- type: f1 |
|
|
value: 73.86147988001356 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (mn) |
|
|
config: mn |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 70.0168123739072 |
|
|
- type: f1 |
|
|
value: 69.38515920054571 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ms) |
|
|
config: ms |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.41156691324814 |
|
|
- type: f1 |
|
|
value: 73.43474953408237 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (my) |
|
|
config: my |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 68.39609952925353 |
|
|
- type: f1 |
|
|
value: 67.29731681109291 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (nb) |
|
|
config: nb |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.20914593140552 |
|
|
- type: f1 |
|
|
value: 77.07066497935367 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (nl) |
|
|
config: nl |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 78.52387357094821 |
|
|
- type: f1 |
|
|
value: 78.5259569473291 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (pl) |
|
|
config: pl |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.6913248150639 |
|
|
- type: f1 |
|
|
value: 76.91201656350455 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (pt) |
|
|
config: pt |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.1217215870881 |
|
|
- type: f1 |
|
|
value: 77.41179937912504 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ro) |
|
|
config: ro |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.25891055817083 |
|
|
- type: f1 |
|
|
value: 75.8089244542887 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ru) |
|
|
config: ru |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.70679219905851 |
|
|
- type: f1 |
|
|
value: 78.21459594517711 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (sl) |
|
|
config: sl |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.83523873570948 |
|
|
- type: f1 |
|
|
value: 74.86847028401978 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (sq) |
|
|
config: sq |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.71755211835911 |
|
|
- type: f1 |
|
|
value: 74.0214326485662 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (sv) |
|
|
config: sv |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 79.06523201075991 |
|
|
- type: f1 |
|
|
value: 79.10545620325138 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (sw) |
|
|
config: sw |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 67.91862811028918 |
|
|
- type: f1 |
|
|
value: 66.50386121217983 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ta) |
|
|
config: ta |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 70.93140551445865 |
|
|
- type: f1 |
|
|
value: 70.755435928495 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (te) |
|
|
config: te |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.40753194351042 |
|
|
- type: f1 |
|
|
value: 71.61816115782923 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (th) |
|
|
config: th |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.1815736381977 |
|
|
- type: f1 |
|
|
value: 75.08016717887205 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (tl) |
|
|
config: tl |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.86482851378614 |
|
|
- type: f1 |
|
|
value: 72.39521180006291 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (tr) |
|
|
config: tr |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.46940147948891 |
|
|
- type: f1 |
|
|
value: 76.70044085362349 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (ur) |
|
|
config: ur |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.89307330195024 |
|
|
- type: f1 |
|
|
value: 71.5721825332298 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (vi) |
|
|
config: vi |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.7511768661735 |
|
|
- type: f1 |
|
|
value: 75.17918654541515 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
|
config: zh-CN |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 78.69535978480162 |
|
|
- type: f1 |
|
|
value: 78.90019070153316 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/amazon_massive_scenario |
|
|
name: MTEB MassiveScenarioClassification (zh-TW) |
|
|
config: zh-TW |
|
|
split: test |
|
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.45729657027572 |
|
|
- type: f1 |
|
|
value: 76.19578371794672 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/medrxiv-clustering-p2p |
|
|
name: MTEB MedrxivClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 36.92715354123554 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/medrxiv-clustering-s2s |
|
|
name: MTEB MedrxivClusteringS2S |
|
|
config: default |
|
|
split: test |
|
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 35.53536244162518 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/mind_small |
|
|
name: MTEB MindSmallReranking |
|
|
config: default |
|
|
split: test |
|
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 33.08507884504006 |
|
|
- type: mrr |
|
|
value: 34.32436977159129 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: nfcorpus |
|
|
name: MTEB NFCorpus |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 5.935 |
|
|
- type: map_at_10 |
|
|
value: 13.297 |
|
|
- type: map_at_100 |
|
|
value: 16.907 |
|
|
- type: map_at_1000 |
|
|
value: 18.391 |
|
|
- type: map_at_3 |
|
|
value: 9.626999999999999 |
|
|
- type: map_at_5 |
|
|
value: 11.190999999999999 |
|
|
- type: mrr_at_1 |
|
|
value: 46.129999999999995 |
|
|
- type: mrr_at_10 |
|
|
value: 54.346000000000004 |
|
|
- type: mrr_at_100 |
|
|
value: 55.067 |
|
|
- type: mrr_at_1000 |
|
|
value: 55.1 |
|
|
- type: mrr_at_3 |
|
|
value: 51.961 |
|
|
- type: mrr_at_5 |
|
|
value: 53.246 |
|
|
- type: ndcg_at_1 |
|
|
value: 44.118 |
|
|
- type: ndcg_at_10 |
|
|
value: 35.534 |
|
|
- type: ndcg_at_100 |
|
|
value: 32.946999999999996 |
|
|
- type: ndcg_at_1000 |
|
|
value: 41.599000000000004 |
|
|
- type: ndcg_at_3 |
|
|
value: 40.25 |
|
|
- type: ndcg_at_5 |
|
|
value: 37.978 |
|
|
- type: precision_at_1 |
|
|
value: 46.129999999999995 |
|
|
- type: precision_at_10 |
|
|
value: 26.842 |
|
|
- type: precision_at_100 |
|
|
value: 8.427 |
|
|
- type: precision_at_1000 |
|
|
value: 2.128 |
|
|
- type: precision_at_3 |
|
|
value: 37.977 |
|
|
- type: precision_at_5 |
|
|
value: 32.879000000000005 |
|
|
- type: recall_at_1 |
|
|
value: 5.935 |
|
|
- type: recall_at_10 |
|
|
value: 17.211000000000002 |
|
|
- type: recall_at_100 |
|
|
value: 34.33 |
|
|
- type: recall_at_1000 |
|
|
value: 65.551 |
|
|
- type: recall_at_3 |
|
|
value: 10.483 |
|
|
- type: recall_at_5 |
|
|
value: 13.078999999999999 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: nq |
|
|
name: MTEB NQ |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 35.231 |
|
|
- type: map_at_10 |
|
|
value: 50.202000000000005 |
|
|
- type: map_at_100 |
|
|
value: 51.154999999999994 |
|
|
- type: map_at_1000 |
|
|
value: 51.181 |
|
|
- type: map_at_3 |
|
|
value: 45.774 |
|
|
- type: map_at_5 |
|
|
value: 48.522 |
|
|
- type: mrr_at_1 |
|
|
value: 39.687 |
|
|
- type: mrr_at_10 |
|
|
value: 52.88 |
|
|
- type: mrr_at_100 |
|
|
value: 53.569 |
|
|
- type: mrr_at_1000 |
|
|
value: 53.58500000000001 |
|
|
- type: mrr_at_3 |
|
|
value: 49.228 |
|
|
- type: mrr_at_5 |
|
|
value: 51.525 |
|
|
- type: ndcg_at_1 |
|
|
value: 39.687 |
|
|
- type: ndcg_at_10 |
|
|
value: 57.754000000000005 |
|
|
- type: ndcg_at_100 |
|
|
value: 61.597 |
|
|
- type: ndcg_at_1000 |
|
|
value: 62.18900000000001 |
|
|
- type: ndcg_at_3 |
|
|
value: 49.55 |
|
|
- type: ndcg_at_5 |
|
|
value: 54.11899999999999 |
|
|
- type: precision_at_1 |
|
|
value: 39.687 |
|
|
- type: precision_at_10 |
|
|
value: 9.313 |
|
|
- type: precision_at_100 |
|
|
value: 1.146 |
|
|
- type: precision_at_1000 |
|
|
value: 0.12 |
|
|
- type: precision_at_3 |
|
|
value: 22.229 |
|
|
- type: precision_at_5 |
|
|
value: 15.939 |
|
|
- type: recall_at_1 |
|
|
value: 35.231 |
|
|
- type: recall_at_10 |
|
|
value: 78.083 |
|
|
- type: recall_at_100 |
|
|
value: 94.42099999999999 |
|
|
- type: recall_at_1000 |
|
|
value: 98.81 |
|
|
- type: recall_at_3 |
|
|
value: 57.047000000000004 |
|
|
- type: recall_at_5 |
|
|
value: 67.637 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: quora |
|
|
name: MTEB QuoraRetrieval |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 71.241 |
|
|
- type: map_at_10 |
|
|
value: 85.462 |
|
|
- type: map_at_100 |
|
|
value: 86.083 |
|
|
- type: map_at_1000 |
|
|
value: 86.09700000000001 |
|
|
- type: map_at_3 |
|
|
value: 82.49499999999999 |
|
|
- type: map_at_5 |
|
|
value: 84.392 |
|
|
- type: mrr_at_1 |
|
|
value: 82.09 |
|
|
- type: mrr_at_10 |
|
|
value: 88.301 |
|
|
- type: mrr_at_100 |
|
|
value: 88.383 |
|
|
- type: mrr_at_1000 |
|
|
value: 88.384 |
|
|
- type: mrr_at_3 |
|
|
value: 87.37 |
|
|
- type: mrr_at_5 |
|
|
value: 88.035 |
|
|
- type: ndcg_at_1 |
|
|
value: 82.12 |
|
|
- type: ndcg_at_10 |
|
|
value: 89.149 |
|
|
- type: ndcg_at_100 |
|
|
value: 90.235 |
|
|
- type: ndcg_at_1000 |
|
|
value: 90.307 |
|
|
- type: ndcg_at_3 |
|
|
value: 86.37599999999999 |
|
|
- type: ndcg_at_5 |
|
|
value: 87.964 |
|
|
- type: precision_at_1 |
|
|
value: 82.12 |
|
|
- type: precision_at_10 |
|
|
value: 13.56 |
|
|
- type: precision_at_100 |
|
|
value: 1.539 |
|
|
- type: precision_at_1000 |
|
|
value: 0.157 |
|
|
- type: precision_at_3 |
|
|
value: 37.88 |
|
|
- type: precision_at_5 |
|
|
value: 24.92 |
|
|
- type: recall_at_1 |
|
|
value: 71.241 |
|
|
- type: recall_at_10 |
|
|
value: 96.128 |
|
|
- type: recall_at_100 |
|
|
value: 99.696 |
|
|
- type: recall_at_1000 |
|
|
value: 99.994 |
|
|
- type: recall_at_3 |
|
|
value: 88.181 |
|
|
- type: recall_at_5 |
|
|
value: 92.694 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/reddit-clustering |
|
|
name: MTEB RedditClustering |
|
|
config: default |
|
|
split: test |
|
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 56.59757799655151 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/reddit-clustering-p2p |
|
|
name: MTEB RedditClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: 282350215ef01743dc01b456c7f5241fa8937f16 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 64.27391998854624 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: scidocs |
|
|
name: MTEB SCIDOCS |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 4.243 |
|
|
- type: map_at_10 |
|
|
value: 10.965 |
|
|
- type: map_at_100 |
|
|
value: 12.934999999999999 |
|
|
- type: map_at_1000 |
|
|
value: 13.256 |
|
|
- type: map_at_3 |
|
|
value: 7.907 |
|
|
- type: map_at_5 |
|
|
value: 9.435 |
|
|
- type: mrr_at_1 |
|
|
value: 20.9 |
|
|
- type: mrr_at_10 |
|
|
value: 31.849 |
|
|
- type: mrr_at_100 |
|
|
value: 32.964 |
|
|
- type: mrr_at_1000 |
|
|
value: 33.024 |
|
|
- type: mrr_at_3 |
|
|
value: 28.517 |
|
|
- type: mrr_at_5 |
|
|
value: 30.381999999999998 |
|
|
- type: ndcg_at_1 |
|
|
value: 20.9 |
|
|
- type: ndcg_at_10 |
|
|
value: 18.723 |
|
|
- type: ndcg_at_100 |
|
|
value: 26.384999999999998 |
|
|
- type: ndcg_at_1000 |
|
|
value: 32.114 |
|
|
- type: ndcg_at_3 |
|
|
value: 17.753 |
|
|
- type: ndcg_at_5 |
|
|
value: 15.558 |
|
|
- type: precision_at_1 |
|
|
value: 20.9 |
|
|
- type: precision_at_10 |
|
|
value: 9.8 |
|
|
- type: precision_at_100 |
|
|
value: 2.078 |
|
|
- type: precision_at_1000 |
|
|
value: 0.345 |
|
|
- type: precision_at_3 |
|
|
value: 16.900000000000002 |
|
|
- type: precision_at_5 |
|
|
value: 13.88 |
|
|
- type: recall_at_1 |
|
|
value: 4.243 |
|
|
- type: recall_at_10 |
|
|
value: 19.885 |
|
|
- type: recall_at_100 |
|
|
value: 42.17 |
|
|
- type: recall_at_1000 |
|
|
value: 70.12 |
|
|
- type: recall_at_3 |
|
|
value: 10.288 |
|
|
- type: recall_at_5 |
|
|
value: 14.072000000000001 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sickr-sts |
|
|
name: MTEB SICK-R |
|
|
config: default |
|
|
split: test |
|
|
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 85.84209174935282 |
|
|
- type: cos_sim_spearman |
|
|
value: 81.73248048438833 |
|
|
- type: euclidean_pearson |
|
|
value: 83.02810070308149 |
|
|
- type: euclidean_spearman |
|
|
value: 81.73248295679514 |
|
|
- type: manhattan_pearson |
|
|
value: 82.95368060376002 |
|
|
- type: manhattan_spearman |
|
|
value: 81.60277910998718 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts12-sts |
|
|
name: MTEB STS12 |
|
|
config: default |
|
|
split: test |
|
|
revision: a0d554a64d88156834ff5ae9920b964011b16384 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 88.52628804556943 |
|
|
- type: cos_sim_spearman |
|
|
value: 82.5713913555672 |
|
|
- type: euclidean_pearson |
|
|
value: 85.8796774746988 |
|
|
- type: euclidean_spearman |
|
|
value: 82.57137506803424 |
|
|
- type: manhattan_pearson |
|
|
value: 85.79671002960058 |
|
|
- type: manhattan_spearman |
|
|
value: 82.49445981618027 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts13-sts |
|
|
name: MTEB STS13 |
|
|
config: default |
|
|
split: test |
|
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 86.23682503505542 |
|
|
- type: cos_sim_spearman |
|
|
value: 87.15008956711806 |
|
|
- type: euclidean_pearson |
|
|
value: 86.79805401524959 |
|
|
- type: euclidean_spearman |
|
|
value: 87.15008956711806 |
|
|
- type: manhattan_pearson |
|
|
value: 86.65298502699244 |
|
|
- type: manhattan_spearman |
|
|
value: 86.97677821948562 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts14-sts |
|
|
name: MTEB STS14 |
|
|
config: default |
|
|
split: test |
|
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 85.63370304677802 |
|
|
- type: cos_sim_spearman |
|
|
value: 84.97105553540318 |
|
|
- type: euclidean_pearson |
|
|
value: 85.28896108687721 |
|
|
- type: euclidean_spearman |
|
|
value: 84.97105553540318 |
|
|
- type: manhattan_pearson |
|
|
value: 85.09663190337331 |
|
|
- type: manhattan_spearman |
|
|
value: 84.79126831644619 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts15-sts |
|
|
name: MTEB STS15 |
|
|
config: default |
|
|
split: test |
|
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 90.2614838800733 |
|
|
- type: cos_sim_spearman |
|
|
value: 91.0509162991835 |
|
|
- type: euclidean_pearson |
|
|
value: 90.33098317533373 |
|
|
- type: euclidean_spearman |
|
|
value: 91.05091625871644 |
|
|
- type: manhattan_pearson |
|
|
value: 90.26250435151107 |
|
|
- type: manhattan_spearman |
|
|
value: 90.97999594417519 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts16-sts |
|
|
name: MTEB STS16 |
|
|
config: default |
|
|
split: test |
|
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 85.80480973335091 |
|
|
- type: cos_sim_spearman |
|
|
value: 87.313695492969 |
|
|
- type: euclidean_pearson |
|
|
value: 86.49267251576939 |
|
|
- type: euclidean_spearman |
|
|
value: 87.313695492969 |
|
|
- type: manhattan_pearson |
|
|
value: 86.44019901831935 |
|
|
- type: manhattan_spearman |
|
|
value: 87.24205395460392 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts17-crosslingual-sts |
|
|
name: MTEB STS17 (en-en) |
|
|
config: en-en |
|
|
split: test |
|
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 90.05662789380672 |
|
|
- type: cos_sim_spearman |
|
|
value: 90.02759424426651 |
|
|
- type: euclidean_pearson |
|
|
value: 90.4042483422981 |
|
|
- type: euclidean_spearman |
|
|
value: 90.02759424426651 |
|
|
- type: manhattan_pearson |
|
|
value: 90.51446975000226 |
|
|
- type: manhattan_spearman |
|
|
value: 90.08832889933616 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/sts22-crosslingual-sts |
|
|
name: MTEB STS22 (en) |
|
|
config: en |
|
|
split: test |
|
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 67.5975528273532 |
|
|
- type: cos_sim_spearman |
|
|
value: 67.62969861411354 |
|
|
- type: euclidean_pearson |
|
|
value: 69.224275734323 |
|
|
- type: euclidean_spearman |
|
|
value: 67.62969861411354 |
|
|
- type: manhattan_pearson |
|
|
value: 69.3761447059927 |
|
|
- type: manhattan_spearman |
|
|
value: 67.90921005611467 |
|
|
- task: |
|
|
type: STS |
|
|
dataset: |
|
|
type: mteb/stsbenchmark-sts |
|
|
name: MTEB STSBenchmark |
|
|
config: default |
|
|
split: test |
|
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 87.11244327231684 |
|
|
- type: cos_sim_spearman |
|
|
value: 88.37902438979035 |
|
|
- type: euclidean_pearson |
|
|
value: 87.86054279847336 |
|
|
- type: euclidean_spearman |
|
|
value: 88.37902438979035 |
|
|
- type: manhattan_pearson |
|
|
value: 87.77257757320378 |
|
|
- type: manhattan_spearman |
|
|
value: 88.25208966098123 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/scidocs-reranking |
|
|
name: MTEB SciDocsRR |
|
|
config: default |
|
|
split: test |
|
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
|
|
metrics: |
|
|
- type: map |
|
|
value: 85.87174608143563 |
|
|
- type: mrr |
|
|
value: 96.12836872640794 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: scifact |
|
|
name: MTEB SciFact |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 57.760999999999996 |
|
|
- type: map_at_10 |
|
|
value: 67.258 |
|
|
- type: map_at_100 |
|
|
value: 67.757 |
|
|
- type: map_at_1000 |
|
|
value: 67.78800000000001 |
|
|
- type: map_at_3 |
|
|
value: 64.602 |
|
|
- type: map_at_5 |
|
|
value: 65.64 |
|
|
- type: mrr_at_1 |
|
|
value: 60.667 |
|
|
- type: mrr_at_10 |
|
|
value: 68.441 |
|
|
- type: mrr_at_100 |
|
|
value: 68.825 |
|
|
- type: mrr_at_1000 |
|
|
value: 68.853 |
|
|
- type: mrr_at_3 |
|
|
value: 66.444 |
|
|
- type: mrr_at_5 |
|
|
value: 67.26100000000001 |
|
|
- type: ndcg_at_1 |
|
|
value: 60.667 |
|
|
- type: ndcg_at_10 |
|
|
value: 71.852 |
|
|
- type: ndcg_at_100 |
|
|
value: 73.9 |
|
|
- type: ndcg_at_1000 |
|
|
value: 74.628 |
|
|
- type: ndcg_at_3 |
|
|
value: 67.093 |
|
|
- type: ndcg_at_5 |
|
|
value: 68.58 |
|
|
- type: precision_at_1 |
|
|
value: 60.667 |
|
|
- type: precision_at_10 |
|
|
value: 9.6 |
|
|
- type: precision_at_100 |
|
|
value: 1.0670000000000002 |
|
|
- type: precision_at_1000 |
|
|
value: 0.11199999999999999 |
|
|
- type: precision_at_3 |
|
|
value: 26.111 |
|
|
- type: precision_at_5 |
|
|
value: 16.733 |
|
|
- type: recall_at_1 |
|
|
value: 57.760999999999996 |
|
|
- type: recall_at_10 |
|
|
value: 84.967 |
|
|
- type: recall_at_100 |
|
|
value: 93.833 |
|
|
- type: recall_at_1000 |
|
|
value: 99.333 |
|
|
- type: recall_at_3 |
|
|
value: 71.589 |
|
|
- type: recall_at_5 |
|
|
value: 75.483 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: mteb/sprintduplicatequestions-pairclassification |
|
|
name: MTEB SprintDuplicateQuestions |
|
|
config: default |
|
|
split: test |
|
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 99.66633663366336 |
|
|
- type: cos_sim_ap |
|
|
value: 91.17685358899108 |
|
|
- type: cos_sim_f1 |
|
|
value: 82.16818642350559 |
|
|
- type: cos_sim_precision |
|
|
value: 83.26488706365504 |
|
|
- type: cos_sim_recall |
|
|
value: 81.10000000000001 |
|
|
- type: dot_accuracy |
|
|
value: 99.66633663366336 |
|
|
- type: dot_ap |
|
|
value: 91.17663411119032 |
|
|
- type: dot_f1 |
|
|
value: 82.16818642350559 |
|
|
- type: dot_precision |
|
|
value: 83.26488706365504 |
|
|
- type: dot_recall |
|
|
value: 81.10000000000001 |
|
|
- type: euclidean_accuracy |
|
|
value: 99.66633663366336 |
|
|
- type: euclidean_ap |
|
|
value: 91.17685189882275 |
|
|
- type: euclidean_f1 |
|
|
value: 82.16818642350559 |
|
|
- type: euclidean_precision |
|
|
value: 83.26488706365504 |
|
|
- type: euclidean_recall |
|
|
value: 81.10000000000001 |
|
|
- type: manhattan_accuracy |
|
|
value: 99.66633663366336 |
|
|
- type: manhattan_ap |
|
|
value: 91.2241619496737 |
|
|
- type: manhattan_f1 |
|
|
value: 82.20472440944883 |
|
|
- type: manhattan_precision |
|
|
value: 86.51933701657458 |
|
|
- type: manhattan_recall |
|
|
value: 78.3 |
|
|
- type: max_accuracy |
|
|
value: 99.66633663366336 |
|
|
- type: max_ap |
|
|
value: 91.2241619496737 |
|
|
- type: max_f1 |
|
|
value: 82.20472440944883 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/stackexchange-clustering |
|
|
name: MTEB StackExchangeClustering |
|
|
config: default |
|
|
split: test |
|
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 66.85101268897951 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/stackexchange-clustering-p2p |
|
|
name: MTEB StackExchangeClusteringP2P |
|
|
config: default |
|
|
split: test |
|
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 42.461184054706905 |
|
|
- task: |
|
|
type: Reranking |
|
|
dataset: |
|
|
type: mteb/stackoverflowdupquestions-reranking |
|
|
name: MTEB StackOverflowDupQuestions |
|
|
config: default |
|
|
split: test |
|
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
|
|
metrics: |
|
|
- type: map |
|
|
value: 51.44542568873886 |
|
|
- type: mrr |
|
|
value: 52.33656151854681 |
|
|
- task: |
|
|
type: Summarization |
|
|
dataset: |
|
|
type: mteb/summeval |
|
|
name: MTEB SummEval |
|
|
config: default |
|
|
split: test |
|
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
|
|
metrics: |
|
|
- type: cos_sim_pearson |
|
|
value: 30.75982974997539 |
|
|
- type: cos_sim_spearman |
|
|
value: 30.385405026539914 |
|
|
- type: dot_pearson |
|
|
value: 30.75982433546523 |
|
|
- type: dot_spearman |
|
|
value: 30.385405026539914 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: trec-covid |
|
|
name: MTEB TRECCOVID |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 0.22799999999999998 |
|
|
- type: map_at_10 |
|
|
value: 2.064 |
|
|
- type: map_at_100 |
|
|
value: 13.056000000000001 |
|
|
- type: map_at_1000 |
|
|
value: 31.747999999999998 |
|
|
- type: map_at_3 |
|
|
value: 0.67 |
|
|
- type: map_at_5 |
|
|
value: 1.097 |
|
|
- type: mrr_at_1 |
|
|
value: 90.0 |
|
|
- type: mrr_at_10 |
|
|
value: 94.667 |
|
|
- type: mrr_at_100 |
|
|
value: 94.667 |
|
|
- type: mrr_at_1000 |
|
|
value: 94.667 |
|
|
- type: mrr_at_3 |
|
|
value: 94.667 |
|
|
- type: mrr_at_5 |
|
|
value: 94.667 |
|
|
- type: ndcg_at_1 |
|
|
value: 86.0 |
|
|
- type: ndcg_at_10 |
|
|
value: 82.0 |
|
|
- type: ndcg_at_100 |
|
|
value: 64.307 |
|
|
- type: ndcg_at_1000 |
|
|
value: 57.023999999999994 |
|
|
- type: ndcg_at_3 |
|
|
value: 85.816 |
|
|
- type: ndcg_at_5 |
|
|
value: 84.904 |
|
|
- type: precision_at_1 |
|
|
value: 90.0 |
|
|
- type: precision_at_10 |
|
|
value: 85.8 |
|
|
- type: precision_at_100 |
|
|
value: 66.46 |
|
|
- type: precision_at_1000 |
|
|
value: 25.202 |
|
|
- type: precision_at_3 |
|
|
value: 90.0 |
|
|
- type: precision_at_5 |
|
|
value: 89.2 |
|
|
- type: recall_at_1 |
|
|
value: 0.22799999999999998 |
|
|
- type: recall_at_10 |
|
|
value: 2.235 |
|
|
- type: recall_at_100 |
|
|
value: 16.185 |
|
|
- type: recall_at_1000 |
|
|
value: 53.620999999999995 |
|
|
- type: recall_at_3 |
|
|
value: 0.7040000000000001 |
|
|
- type: recall_at_5 |
|
|
value: 1.172 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (sqi-eng) |
|
|
config: sqi-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.39999999999999 |
|
|
- type: f1 |
|
|
value: 96.75 |
|
|
- type: precision |
|
|
value: 96.45 |
|
|
- type: recall |
|
|
value: 97.39999999999999 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (fry-eng) |
|
|
config: fry-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 85.54913294797689 |
|
|
- type: f1 |
|
|
value: 82.46628131021194 |
|
|
- type: precision |
|
|
value: 81.1175337186898 |
|
|
- type: recall |
|
|
value: 85.54913294797689 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (kur-eng) |
|
|
config: kur-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 81.21951219512195 |
|
|
- type: f1 |
|
|
value: 77.33333333333334 |
|
|
- type: precision |
|
|
value: 75.54878048780488 |
|
|
- type: recall |
|
|
value: 81.21951219512195 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tur-eng) |
|
|
config: tur-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 98.6 |
|
|
- type: f1 |
|
|
value: 98.26666666666665 |
|
|
- type: precision |
|
|
value: 98.1 |
|
|
- type: recall |
|
|
value: 98.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (deu-eng) |
|
|
config: deu-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 99.5 |
|
|
- type: f1 |
|
|
value: 99.33333333333333 |
|
|
- type: precision |
|
|
value: 99.25 |
|
|
- type: recall |
|
|
value: 99.5 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (nld-eng) |
|
|
config: nld-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.8 |
|
|
- type: f1 |
|
|
value: 97.2 |
|
|
- type: precision |
|
|
value: 96.89999999999999 |
|
|
- type: recall |
|
|
value: 97.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ron-eng) |
|
|
config: ron-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.8 |
|
|
- type: f1 |
|
|
value: 97.18333333333334 |
|
|
- type: precision |
|
|
value: 96.88333333333333 |
|
|
- type: recall |
|
|
value: 97.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ang-eng) |
|
|
config: ang-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.61194029850746 |
|
|
- type: f1 |
|
|
value: 72.81094527363183 |
|
|
- type: precision |
|
|
value: 70.83333333333333 |
|
|
- type: recall |
|
|
value: 77.61194029850746 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ido-eng) |
|
|
config: ido-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.7 |
|
|
- type: f1 |
|
|
value: 91.91666666666667 |
|
|
- type: precision |
|
|
value: 91.08333333333334 |
|
|
- type: recall |
|
|
value: 93.7 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (jav-eng) |
|
|
config: jav-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 88.29268292682927 |
|
|
- type: f1 |
|
|
value: 85.27642276422765 |
|
|
- type: precision |
|
|
value: 84.01277584204414 |
|
|
- type: recall |
|
|
value: 88.29268292682927 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (isl-eng) |
|
|
config: isl-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.1 |
|
|
- type: f1 |
|
|
value: 95.0 |
|
|
- type: precision |
|
|
value: 94.46666666666668 |
|
|
- type: recall |
|
|
value: 96.1 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (slv-eng) |
|
|
config: slv-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.681652490887 |
|
|
- type: f1 |
|
|
value: 91.90765492102065 |
|
|
- type: precision |
|
|
value: 91.05913325232888 |
|
|
- type: recall |
|
|
value: 93.681652490887 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (cym-eng) |
|
|
config: cym-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 92.17391304347827 |
|
|
- type: f1 |
|
|
value: 89.97101449275361 |
|
|
- type: precision |
|
|
value: 88.96811594202899 |
|
|
- type: recall |
|
|
value: 92.17391304347827 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (kaz-eng) |
|
|
config: kaz-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 90.43478260869566 |
|
|
- type: f1 |
|
|
value: 87.72173913043478 |
|
|
- type: precision |
|
|
value: 86.42028985507245 |
|
|
- type: recall |
|
|
value: 90.43478260869566 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (est-eng) |
|
|
config: est-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 90.4 |
|
|
- type: f1 |
|
|
value: 88.03 |
|
|
- type: precision |
|
|
value: 86.95 |
|
|
- type: recall |
|
|
value: 90.4 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (heb-eng) |
|
|
config: heb-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.4 |
|
|
- type: f1 |
|
|
value: 91.45666666666666 |
|
|
- type: precision |
|
|
value: 90.525 |
|
|
- type: recall |
|
|
value: 93.4 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (gla-eng) |
|
|
config: gla-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 81.9059107358263 |
|
|
- type: f1 |
|
|
value: 78.32557872364869 |
|
|
- type: precision |
|
|
value: 76.78260286824823 |
|
|
- type: recall |
|
|
value: 81.9059107358263 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (mar-eng) |
|
|
config: mar-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.3 |
|
|
- type: f1 |
|
|
value: 92.58333333333333 |
|
|
- type: precision |
|
|
value: 91.73333333333332 |
|
|
- type: recall |
|
|
value: 94.3 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (lat-eng) |
|
|
config: lat-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 79.10000000000001 |
|
|
- type: f1 |
|
|
value: 74.50500000000001 |
|
|
- type: precision |
|
|
value: 72.58928571428571 |
|
|
- type: recall |
|
|
value: 79.10000000000001 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (bel-eng) |
|
|
config: bel-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.6 |
|
|
- type: f1 |
|
|
value: 95.55 |
|
|
- type: precision |
|
|
value: 95.05 |
|
|
- type: recall |
|
|
value: 96.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (pms-eng) |
|
|
config: pms-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 82.0952380952381 |
|
|
- type: f1 |
|
|
value: 77.98458049886621 |
|
|
- type: precision |
|
|
value: 76.1968253968254 |
|
|
- type: recall |
|
|
value: 82.0952380952381 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (gle-eng) |
|
|
config: gle-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 87.9 |
|
|
- type: f1 |
|
|
value: 84.99190476190476 |
|
|
- type: precision |
|
|
value: 83.65 |
|
|
- type: recall |
|
|
value: 87.9 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (pes-eng) |
|
|
config: pes-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.7 |
|
|
- type: f1 |
|
|
value: 94.56666666666666 |
|
|
- type: precision |
|
|
value: 94.01666666666667 |
|
|
- type: recall |
|
|
value: 95.7 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (nob-eng) |
|
|
config: nob-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 98.6 |
|
|
- type: f1 |
|
|
value: 98.2 |
|
|
- type: precision |
|
|
value: 98.0 |
|
|
- type: recall |
|
|
value: 98.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (bul-eng) |
|
|
config: bul-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.6 |
|
|
- type: f1 |
|
|
value: 94.38333333333334 |
|
|
- type: precision |
|
|
value: 93.78333333333335 |
|
|
- type: recall |
|
|
value: 95.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (cbk-eng) |
|
|
config: cbk-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 87.4 |
|
|
- type: f1 |
|
|
value: 84.10380952380952 |
|
|
- type: precision |
|
|
value: 82.67 |
|
|
- type: recall |
|
|
value: 87.4 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (hun-eng) |
|
|
config: hun-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.5 |
|
|
- type: f1 |
|
|
value: 94.33333333333334 |
|
|
- type: precision |
|
|
value: 93.78333333333333 |
|
|
- type: recall |
|
|
value: 95.5 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (uig-eng) |
|
|
config: uig-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 89.4 |
|
|
- type: f1 |
|
|
value: 86.82000000000001 |
|
|
- type: precision |
|
|
value: 85.64500000000001 |
|
|
- type: recall |
|
|
value: 89.4 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (rus-eng) |
|
|
config: rus-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.1 |
|
|
- type: f1 |
|
|
value: 93.56666666666668 |
|
|
- type: precision |
|
|
value: 92.81666666666666 |
|
|
- type: recall |
|
|
value: 95.1 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (spa-eng) |
|
|
config: spa-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 98.9 |
|
|
- type: f1 |
|
|
value: 98.6 |
|
|
- type: precision |
|
|
value: 98.45 |
|
|
- type: recall |
|
|
value: 98.9 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (hye-eng) |
|
|
config: hye-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.01347708894879 |
|
|
- type: f1 |
|
|
value: 93.51752021563343 |
|
|
- type: precision |
|
|
value: 92.82794249775381 |
|
|
- type: recall |
|
|
value: 95.01347708894879 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tel-eng) |
|
|
config: tel-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.00854700854701 |
|
|
- type: f1 |
|
|
value: 96.08262108262107 |
|
|
- type: precision |
|
|
value: 95.65527065527067 |
|
|
- type: recall |
|
|
value: 97.00854700854701 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (afr-eng) |
|
|
config: afr-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.5 |
|
|
- type: f1 |
|
|
value: 95.39999999999999 |
|
|
- type: precision |
|
|
value: 94.88333333333333 |
|
|
- type: recall |
|
|
value: 96.5 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (mon-eng) |
|
|
config: mon-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.5909090909091 |
|
|
- type: f1 |
|
|
value: 95.49242424242425 |
|
|
- type: precision |
|
|
value: 94.9621212121212 |
|
|
- type: recall |
|
|
value: 96.5909090909091 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (arz-eng) |
|
|
config: arz-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 84.90566037735849 |
|
|
- type: f1 |
|
|
value: 81.85883997204752 |
|
|
- type: precision |
|
|
value: 80.54507337526205 |
|
|
- type: recall |
|
|
value: 84.90566037735849 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (hrv-eng) |
|
|
config: hrv-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.5 |
|
|
- type: f1 |
|
|
value: 96.75 |
|
|
- type: precision |
|
|
value: 96.38333333333333 |
|
|
- type: recall |
|
|
value: 97.5 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (nov-eng) |
|
|
config: nov-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 86.7704280155642 |
|
|
- type: f1 |
|
|
value: 82.99610894941635 |
|
|
- type: precision |
|
|
value: 81.32295719844358 |
|
|
- type: recall |
|
|
value: 86.7704280155642 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (gsw-eng) |
|
|
config: gsw-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 67.52136752136752 |
|
|
- type: f1 |
|
|
value: 61.89662189662191 |
|
|
- type: precision |
|
|
value: 59.68660968660969 |
|
|
- type: recall |
|
|
value: 67.52136752136752 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (nds-eng) |
|
|
config: nds-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 89.2 |
|
|
- type: f1 |
|
|
value: 86.32 |
|
|
- type: precision |
|
|
value: 85.015 |
|
|
- type: recall |
|
|
value: 89.2 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ukr-eng) |
|
|
config: ukr-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.0 |
|
|
- type: f1 |
|
|
value: 94.78333333333333 |
|
|
- type: precision |
|
|
value: 94.18333333333334 |
|
|
- type: recall |
|
|
value: 96.0 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (uzb-eng) |
|
|
config: uzb-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 83.8785046728972 |
|
|
- type: f1 |
|
|
value: 80.54517133956385 |
|
|
- type: precision |
|
|
value: 79.154984423676 |
|
|
- type: recall |
|
|
value: 83.8785046728972 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (lit-eng) |
|
|
config: lit-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.60000000000001 |
|
|
- type: f1 |
|
|
value: 92.01333333333334 |
|
|
- type: precision |
|
|
value: 91.28333333333333 |
|
|
- type: recall |
|
|
value: 93.60000000000001 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ina-eng) |
|
|
config: ina-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.1 |
|
|
- type: f1 |
|
|
value: 96.26666666666667 |
|
|
- type: precision |
|
|
value: 95.85000000000001 |
|
|
- type: recall |
|
|
value: 97.1 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (lfn-eng) |
|
|
config: lfn-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 84.3 |
|
|
- type: f1 |
|
|
value: 80.67833333333333 |
|
|
- type: precision |
|
|
value: 79.03928571428571 |
|
|
- type: recall |
|
|
value: 84.3 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (zsm-eng) |
|
|
config: zsm-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.3 |
|
|
- type: f1 |
|
|
value: 96.48333333333332 |
|
|
- type: precision |
|
|
value: 96.08333333333331 |
|
|
- type: recall |
|
|
value: 97.3 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ita-eng) |
|
|
config: ita-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.7 |
|
|
- type: f1 |
|
|
value: 94.66666666666667 |
|
|
- type: precision |
|
|
value: 94.16666666666667 |
|
|
- type: recall |
|
|
value: 95.7 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (cmn-eng) |
|
|
config: cmn-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.2 |
|
|
- type: f1 |
|
|
value: 96.36666666666667 |
|
|
- type: precision |
|
|
value: 95.96666666666668 |
|
|
- type: recall |
|
|
value: 97.2 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (lvs-eng) |
|
|
config: lvs-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.3 |
|
|
- type: f1 |
|
|
value: 92.80666666666667 |
|
|
- type: precision |
|
|
value: 92.12833333333333 |
|
|
- type: recall |
|
|
value: 94.3 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (glg-eng) |
|
|
config: glg-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.0 |
|
|
- type: f1 |
|
|
value: 96.22333333333334 |
|
|
- type: precision |
|
|
value: 95.875 |
|
|
- type: recall |
|
|
value: 97.0 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ceb-eng) |
|
|
config: ceb-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 74.33333333333333 |
|
|
- type: f1 |
|
|
value: 70.78174603174602 |
|
|
- type: precision |
|
|
value: 69.28333333333332 |
|
|
- type: recall |
|
|
value: 74.33333333333333 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (bre-eng) |
|
|
config: bre-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 37.6 |
|
|
- type: f1 |
|
|
value: 32.938348952090365 |
|
|
- type: precision |
|
|
value: 31.2811038961039 |
|
|
- type: recall |
|
|
value: 37.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ben-eng) |
|
|
config: ben-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 91.5 |
|
|
- type: f1 |
|
|
value: 89.13333333333333 |
|
|
- type: precision |
|
|
value: 88.03333333333333 |
|
|
- type: recall |
|
|
value: 91.5 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (swg-eng) |
|
|
config: swg-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 82.14285714285714 |
|
|
- type: f1 |
|
|
value: 77.67857142857143 |
|
|
- type: precision |
|
|
value: 75.59523809523809 |
|
|
- type: recall |
|
|
value: 82.14285714285714 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (arq-eng) |
|
|
config: arq-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 69.0450054884742 |
|
|
- type: f1 |
|
|
value: 63.070409283362075 |
|
|
- type: precision |
|
|
value: 60.58992781824835 |
|
|
- type: recall |
|
|
value: 69.0450054884742 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (kab-eng) |
|
|
config: kab-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 63.1 |
|
|
- type: f1 |
|
|
value: 57.848333333333336 |
|
|
- type: precision |
|
|
value: 55.69500000000001 |
|
|
- type: recall |
|
|
value: 63.1 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (fra-eng) |
|
|
config: fra-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.1 |
|
|
- type: f1 |
|
|
value: 95.01666666666667 |
|
|
- type: precision |
|
|
value: 94.5 |
|
|
- type: recall |
|
|
value: 96.1 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (por-eng) |
|
|
config: por-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.89999999999999 |
|
|
- type: f1 |
|
|
value: 94.90666666666667 |
|
|
- type: precision |
|
|
value: 94.425 |
|
|
- type: recall |
|
|
value: 95.89999999999999 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tat-eng) |
|
|
config: tat-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 87.6 |
|
|
- type: f1 |
|
|
value: 84.61333333333333 |
|
|
- type: precision |
|
|
value: 83.27 |
|
|
- type: recall |
|
|
value: 87.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (oci-eng) |
|
|
config: oci-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 76.4 |
|
|
- type: f1 |
|
|
value: 71.90746031746032 |
|
|
- type: precision |
|
|
value: 70.07027777777778 |
|
|
- type: recall |
|
|
value: 76.4 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (pol-eng) |
|
|
config: pol-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.89999999999999 |
|
|
- type: f1 |
|
|
value: 97.26666666666667 |
|
|
- type: precision |
|
|
value: 96.95 |
|
|
- type: recall |
|
|
value: 97.89999999999999 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (war-eng) |
|
|
config: war-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 78.8 |
|
|
- type: f1 |
|
|
value: 74.39555555555555 |
|
|
- type: precision |
|
|
value: 72.59416666666667 |
|
|
- type: recall |
|
|
value: 78.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (aze-eng) |
|
|
config: aze-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.19999999999999 |
|
|
- type: f1 |
|
|
value: 93.78999999999999 |
|
|
- type: precision |
|
|
value: 93.125 |
|
|
- type: recall |
|
|
value: 95.19999999999999 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (vie-eng) |
|
|
config: vie-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.8 |
|
|
- type: f1 |
|
|
value: 97.1 |
|
|
- type: precision |
|
|
value: 96.75 |
|
|
- type: recall |
|
|
value: 97.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (nno-eng) |
|
|
config: nno-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.6 |
|
|
- type: f1 |
|
|
value: 94.25666666666666 |
|
|
- type: precision |
|
|
value: 93.64166666666668 |
|
|
- type: recall |
|
|
value: 95.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (cha-eng) |
|
|
config: cha-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 56.934306569343065 |
|
|
- type: f1 |
|
|
value: 51.461591936044485 |
|
|
- type: precision |
|
|
value: 49.37434827945776 |
|
|
- type: recall |
|
|
value: 56.934306569343065 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (mhr-eng) |
|
|
config: mhr-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 20.200000000000003 |
|
|
- type: f1 |
|
|
value: 16.91799284049284 |
|
|
- type: precision |
|
|
value: 15.791855158730158 |
|
|
- type: recall |
|
|
value: 20.200000000000003 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (dan-eng) |
|
|
config: dan-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.2 |
|
|
- type: f1 |
|
|
value: 95.3 |
|
|
- type: precision |
|
|
value: 94.85 |
|
|
- type: recall |
|
|
value: 96.2 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ell-eng) |
|
|
config: ell-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.3 |
|
|
- type: f1 |
|
|
value: 95.11666666666667 |
|
|
- type: precision |
|
|
value: 94.53333333333333 |
|
|
- type: recall |
|
|
value: 96.3 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (amh-eng) |
|
|
config: amh-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 89.88095238095238 |
|
|
- type: f1 |
|
|
value: 87.14285714285714 |
|
|
- type: precision |
|
|
value: 85.96230158730161 |
|
|
- type: recall |
|
|
value: 89.88095238095238 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (pam-eng) |
|
|
config: pam-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 24.099999999999998 |
|
|
- type: f1 |
|
|
value: 19.630969083349783 |
|
|
- type: precision |
|
|
value: 18.275094905094907 |
|
|
- type: recall |
|
|
value: 24.099999999999998 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (hsb-eng) |
|
|
config: hsb-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 83.4368530020704 |
|
|
- type: f1 |
|
|
value: 79.45183870649709 |
|
|
- type: precision |
|
|
value: 77.7432712215321 |
|
|
- type: recall |
|
|
value: 83.4368530020704 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (srp-eng) |
|
|
config: srp-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.8 |
|
|
- type: f1 |
|
|
value: 94.53333333333333 |
|
|
- type: precision |
|
|
value: 93.91666666666666 |
|
|
- type: recall |
|
|
value: 95.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (epo-eng) |
|
|
config: epo-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 98.8 |
|
|
- type: f1 |
|
|
value: 98.48333333333332 |
|
|
- type: precision |
|
|
value: 98.33333333333334 |
|
|
- type: recall |
|
|
value: 98.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (kzj-eng) |
|
|
config: kzj-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 17.5 |
|
|
- type: f1 |
|
|
value: 14.979285714285714 |
|
|
- type: precision |
|
|
value: 14.23235060690943 |
|
|
- type: recall |
|
|
value: 17.5 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (awa-eng) |
|
|
config: awa-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.93939393939394 |
|
|
- type: f1 |
|
|
value: 91.991341991342 |
|
|
- type: precision |
|
|
value: 91.05339105339105 |
|
|
- type: recall |
|
|
value: 93.93939393939394 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (fao-eng) |
|
|
config: fao-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 89.31297709923665 |
|
|
- type: f1 |
|
|
value: 86.76844783715012 |
|
|
- type: precision |
|
|
value: 85.63613231552164 |
|
|
- type: recall |
|
|
value: 89.31297709923665 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (mal-eng) |
|
|
config: mal-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 99.12663755458514 |
|
|
- type: f1 |
|
|
value: 98.93255701115964 |
|
|
- type: precision |
|
|
value: 98.83551673944687 |
|
|
- type: recall |
|
|
value: 99.12663755458514 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ile-eng) |
|
|
config: ile-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 92.0 |
|
|
- type: f1 |
|
|
value: 89.77999999999999 |
|
|
- type: precision |
|
|
value: 88.78333333333333 |
|
|
- type: recall |
|
|
value: 92.0 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (bos-eng) |
|
|
config: bos-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.89265536723164 |
|
|
- type: f1 |
|
|
value: 95.85687382297553 |
|
|
- type: precision |
|
|
value: 95.33898305084746 |
|
|
- type: recall |
|
|
value: 96.89265536723164 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (cor-eng) |
|
|
config: cor-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 14.6 |
|
|
- type: f1 |
|
|
value: 11.820611790170615 |
|
|
- type: precision |
|
|
value: 11.022616224355355 |
|
|
- type: recall |
|
|
value: 14.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (cat-eng) |
|
|
config: cat-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.89999999999999 |
|
|
- type: f1 |
|
|
value: 94.93333333333334 |
|
|
- type: precision |
|
|
value: 94.48666666666666 |
|
|
- type: recall |
|
|
value: 95.89999999999999 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (eus-eng) |
|
|
config: eus-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 87.6 |
|
|
- type: f1 |
|
|
value: 84.72333333333334 |
|
|
- type: precision |
|
|
value: 83.44166666666666 |
|
|
- type: recall |
|
|
value: 87.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (yue-eng) |
|
|
config: yue-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.8 |
|
|
- type: f1 |
|
|
value: 93.47333333333333 |
|
|
- type: precision |
|
|
value: 92.875 |
|
|
- type: recall |
|
|
value: 94.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (swe-eng) |
|
|
config: swe-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.6 |
|
|
- type: f1 |
|
|
value: 95.71666666666665 |
|
|
- type: precision |
|
|
value: 95.28333333333335 |
|
|
- type: recall |
|
|
value: 96.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (dtp-eng) |
|
|
config: dtp-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 17.8 |
|
|
- type: f1 |
|
|
value: 14.511074040901628 |
|
|
- type: precision |
|
|
value: 13.503791000666002 |
|
|
- type: recall |
|
|
value: 17.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (kat-eng) |
|
|
config: kat-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.10187667560321 |
|
|
- type: f1 |
|
|
value: 92.46648793565683 |
|
|
- type: precision |
|
|
value: 91.71134941912423 |
|
|
- type: recall |
|
|
value: 94.10187667560321 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (jpn-eng) |
|
|
config: jpn-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.0 |
|
|
- type: f1 |
|
|
value: 96.11666666666666 |
|
|
- type: precision |
|
|
value: 95.68333333333334 |
|
|
- type: recall |
|
|
value: 97.0 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (csb-eng) |
|
|
config: csb-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 72.72727272727273 |
|
|
- type: f1 |
|
|
value: 66.58949745906267 |
|
|
- type: precision |
|
|
value: 63.86693017127799 |
|
|
- type: recall |
|
|
value: 72.72727272727273 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (xho-eng) |
|
|
config: xho-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 90.14084507042254 |
|
|
- type: f1 |
|
|
value: 88.26291079812206 |
|
|
- type: precision |
|
|
value: 87.32394366197182 |
|
|
- type: recall |
|
|
value: 90.14084507042254 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (orv-eng) |
|
|
config: orv-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 64.67065868263472 |
|
|
- type: f1 |
|
|
value: 58.2876627696987 |
|
|
- type: precision |
|
|
value: 55.79255774165953 |
|
|
- type: recall |
|
|
value: 64.67065868263472 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ind-eng) |
|
|
config: ind-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 95.6 |
|
|
- type: f1 |
|
|
value: 94.41666666666667 |
|
|
- type: precision |
|
|
value: 93.85 |
|
|
- type: recall |
|
|
value: 95.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tuk-eng) |
|
|
config: tuk-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 55.172413793103445 |
|
|
- type: f1 |
|
|
value: 49.63992493549144 |
|
|
- type: precision |
|
|
value: 47.71405113769646 |
|
|
- type: recall |
|
|
value: 55.172413793103445 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (max-eng) |
|
|
config: max-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.46478873239437 |
|
|
- type: f1 |
|
|
value: 73.4417616811983 |
|
|
- type: precision |
|
|
value: 71.91607981220658 |
|
|
- type: recall |
|
|
value: 77.46478873239437 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (swh-eng) |
|
|
config: swh-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 84.61538461538461 |
|
|
- type: f1 |
|
|
value: 80.91452991452994 |
|
|
- type: precision |
|
|
value: 79.33760683760683 |
|
|
- type: recall |
|
|
value: 84.61538461538461 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (hin-eng) |
|
|
config: hin-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 98.2 |
|
|
- type: f1 |
|
|
value: 97.6 |
|
|
- type: precision |
|
|
value: 97.3 |
|
|
- type: recall |
|
|
value: 98.2 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (dsb-eng) |
|
|
config: dsb-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 75.5741127348643 |
|
|
- type: f1 |
|
|
value: 72.00417536534445 |
|
|
- type: precision |
|
|
value: 70.53467872883321 |
|
|
- type: recall |
|
|
value: 75.5741127348643 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ber-eng) |
|
|
config: ber-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 62.2 |
|
|
- type: f1 |
|
|
value: 55.577460317460314 |
|
|
- type: precision |
|
|
value: 52.98583333333333 |
|
|
- type: recall |
|
|
value: 62.2 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tam-eng) |
|
|
config: tam-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 92.18241042345277 |
|
|
- type: f1 |
|
|
value: 90.6468124709167 |
|
|
- type: precision |
|
|
value: 89.95656894679696 |
|
|
- type: recall |
|
|
value: 92.18241042345277 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (slk-eng) |
|
|
config: slk-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.1 |
|
|
- type: f1 |
|
|
value: 95.13333333333333 |
|
|
- type: precision |
|
|
value: 94.66666666666667 |
|
|
- type: recall |
|
|
value: 96.1 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tgl-eng) |
|
|
config: tgl-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 96.8 |
|
|
- type: f1 |
|
|
value: 95.85000000000001 |
|
|
- type: precision |
|
|
value: 95.39999999999999 |
|
|
- type: recall |
|
|
value: 96.8 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ast-eng) |
|
|
config: ast-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 92.1259842519685 |
|
|
- type: f1 |
|
|
value: 89.76377952755905 |
|
|
- type: precision |
|
|
value: 88.71391076115485 |
|
|
- type: recall |
|
|
value: 92.1259842519685 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (mkd-eng) |
|
|
config: mkd-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.1 |
|
|
- type: f1 |
|
|
value: 92.49 |
|
|
- type: precision |
|
|
value: 91.725 |
|
|
- type: recall |
|
|
value: 94.1 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (khm-eng) |
|
|
config: khm-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 77.5623268698061 |
|
|
- type: f1 |
|
|
value: 73.27364463791058 |
|
|
- type: precision |
|
|
value: 71.51947852086357 |
|
|
- type: recall |
|
|
value: 77.5623268698061 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ces-eng) |
|
|
config: ces-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.39999999999999 |
|
|
- type: f1 |
|
|
value: 96.56666666666666 |
|
|
- type: precision |
|
|
value: 96.16666666666667 |
|
|
- type: recall |
|
|
value: 97.39999999999999 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tzl-eng) |
|
|
config: tzl-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 66.34615384615384 |
|
|
- type: f1 |
|
|
value: 61.092032967032964 |
|
|
- type: precision |
|
|
value: 59.27197802197802 |
|
|
- type: recall |
|
|
value: 66.34615384615384 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (urd-eng) |
|
|
config: urd-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.89999999999999 |
|
|
- type: f1 |
|
|
value: 93.41190476190476 |
|
|
- type: precision |
|
|
value: 92.7 |
|
|
- type: recall |
|
|
value: 94.89999999999999 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (ara-eng) |
|
|
config: ara-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.10000000000001 |
|
|
- type: f1 |
|
|
value: 91.10000000000001 |
|
|
- type: precision |
|
|
value: 90.13333333333333 |
|
|
- type: recall |
|
|
value: 93.10000000000001 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (kor-eng) |
|
|
config: kor-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 93.7 |
|
|
- type: f1 |
|
|
value: 91.97333333333334 |
|
|
- type: precision |
|
|
value: 91.14166666666667 |
|
|
- type: recall |
|
|
value: 93.7 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (yid-eng) |
|
|
config: yid-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 92.21698113207547 |
|
|
- type: f1 |
|
|
value: 90.3796046720575 |
|
|
- type: precision |
|
|
value: 89.56367924528303 |
|
|
- type: recall |
|
|
value: 92.21698113207547 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (fin-eng) |
|
|
config: fin-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.6 |
|
|
- type: f1 |
|
|
value: 96.91666666666667 |
|
|
- type: precision |
|
|
value: 96.6 |
|
|
- type: recall |
|
|
value: 97.6 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (tha-eng) |
|
|
config: tha-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 97.44525547445255 |
|
|
- type: f1 |
|
|
value: 96.71532846715328 |
|
|
- type: precision |
|
|
value: 96.35036496350365 |
|
|
- type: recall |
|
|
value: 97.44525547445255 |
|
|
- task: |
|
|
type: BitextMining |
|
|
dataset: |
|
|
type: mteb/tatoeba-bitext-mining |
|
|
name: MTEB Tatoeba (wuu-eng) |
|
|
config: wuu-eng |
|
|
split: test |
|
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 94.1 |
|
|
- type: f1 |
|
|
value: 92.34000000000002 |
|
|
- type: precision |
|
|
value: 91.49166666666667 |
|
|
- type: recall |
|
|
value: 94.1 |
|
|
- task: |
|
|
type: Retrieval |
|
|
dataset: |
|
|
type: webis-touche2020 |
|
|
name: MTEB Touche2020 |
|
|
config: default |
|
|
split: test |
|
|
revision: None |
|
|
metrics: |
|
|
- type: map_at_1 |
|
|
value: 3.2910000000000004 |
|
|
- type: map_at_10 |
|
|
value: 10.373000000000001 |
|
|
- type: map_at_100 |
|
|
value: 15.612 |
|
|
- type: map_at_1000 |
|
|
value: 17.06 |
|
|
- type: map_at_3 |
|
|
value: 6.119 |
|
|
- type: map_at_5 |
|
|
value: 7.917000000000001 |
|
|
- type: mrr_at_1 |
|
|
value: 44.897999999999996 |
|
|
- type: mrr_at_10 |
|
|
value: 56.054 |
|
|
- type: mrr_at_100 |
|
|
value: 56.82000000000001 |
|
|
- type: mrr_at_1000 |
|
|
value: 56.82000000000001 |
|
|
- type: mrr_at_3 |
|
|
value: 52.381 |
|
|
- type: mrr_at_5 |
|
|
value: 53.81 |
|
|
- type: ndcg_at_1 |
|
|
value: 42.857 |
|
|
- type: ndcg_at_10 |
|
|
value: 27.249000000000002 |
|
|
- type: ndcg_at_100 |
|
|
value: 36.529 |
|
|
- type: ndcg_at_1000 |
|
|
value: 48.136 |
|
|
- type: ndcg_at_3 |
|
|
value: 33.938 |
|
|
- type: ndcg_at_5 |
|
|
value: 29.951 |
|
|
- type: precision_at_1 |
|
|
value: 44.897999999999996 |
|
|
- type: precision_at_10 |
|
|
value: 22.653000000000002 |
|
|
- type: precision_at_100 |
|
|
value: 7.000000000000001 |
|
|
- type: precision_at_1000 |
|
|
value: 1.48 |
|
|
- type: precision_at_3 |
|
|
value: 32.653 |
|
|
- type: precision_at_5 |
|
|
value: 27.755000000000003 |
|
|
- type: recall_at_1 |
|
|
value: 3.2910000000000004 |
|
|
- type: recall_at_10 |
|
|
value: 16.16 |
|
|
- type: recall_at_100 |
|
|
value: 43.908 |
|
|
- type: recall_at_1000 |
|
|
value: 79.823 |
|
|
- type: recall_at_3 |
|
|
value: 7.156 |
|
|
- type: recall_at_5 |
|
|
value: 10.204 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/toxic_conversations_50k |
|
|
name: MTEB ToxicConversationsClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 71.05879999999999 |
|
|
- type: ap |
|
|
value: 14.609748142799111 |
|
|
- type: f1 |
|
|
value: 54.878956295843096 |
|
|
- task: |
|
|
type: Classification |
|
|
dataset: |
|
|
type: mteb/tweet_sentiment_extraction |
|
|
name: MTEB TweetSentimentExtractionClassification |
|
|
config: default |
|
|
split: test |
|
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
|
|
metrics: |
|
|
- type: accuracy |
|
|
value: 64.61799660441426 |
|
|
- type: f1 |
|
|
value: 64.8698191961434 |
|
|
- task: |
|
|
type: Clustering |
|
|
dataset: |
|
|
type: mteb/twentynewsgroups-clustering |
|
|
name: MTEB TwentyNewsgroupsClustering |
|
|
config: default |
|
|
split: test |
|
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
|
|
metrics: |
|
|
- type: v_measure |
|
|
value: 51.32860036611885 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: mteb/twittersemeval2015-pairclassification |
|
|
name: MTEB TwitterSemEval2015 |
|
|
config: default |
|
|
split: test |
|
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 88.34714192048638 |
|
|
- type: cos_sim_ap |
|
|
value: 80.26732975975634 |
|
|
- type: cos_sim_f1 |
|
|
value: 73.53415148134374 |
|
|
- type: cos_sim_precision |
|
|
value: 69.34767360299276 |
|
|
- type: cos_sim_recall |
|
|
value: 78.25857519788919 |
|
|
- type: dot_accuracy |
|
|
value: 88.34714192048638 |
|
|
- type: dot_ap |
|
|
value: 80.26733698491206 |
|
|
- type: dot_f1 |
|
|
value: 73.53415148134374 |
|
|
- type: dot_precision |
|
|
value: 69.34767360299276 |
|
|
- type: dot_recall |
|
|
value: 78.25857519788919 |
|
|
- type: euclidean_accuracy |
|
|
value: 88.34714192048638 |
|
|
- type: euclidean_ap |
|
|
value: 80.26734337771738 |
|
|
- type: euclidean_f1 |
|
|
value: 73.53415148134374 |
|
|
- type: euclidean_precision |
|
|
value: 69.34767360299276 |
|
|
- type: euclidean_recall |
|
|
value: 78.25857519788919 |
|
|
- type: manhattan_accuracy |
|
|
value: 88.30541813196639 |
|
|
- type: manhattan_ap |
|
|
value: 80.19415808104145 |
|
|
- type: manhattan_f1 |
|
|
value: 73.55143870713441 |
|
|
- type: manhattan_precision |
|
|
value: 73.25307511122743 |
|
|
- type: manhattan_recall |
|
|
value: 73.85224274406332 |
|
|
- type: max_accuracy |
|
|
value: 88.34714192048638 |
|
|
- type: max_ap |
|
|
value: 80.26734337771738 |
|
|
- type: max_f1 |
|
|
value: 73.55143870713441 |
|
|
- task: |
|
|
type: PairClassification |
|
|
dataset: |
|
|
type: mteb/twitterurlcorpus-pairclassification |
|
|
name: MTEB TwitterURLCorpus |
|
|
config: default |
|
|
split: test |
|
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
|
|
metrics: |
|
|
- type: cos_sim_accuracy |
|
|
value: 89.81061047075717 |
|
|
- type: cos_sim_ap |
|
|
value: 87.11747055081017 |
|
|
- type: cos_sim_f1 |
|
|
value: 80.04355498817256 |
|
|
- type: cos_sim_precision |
|
|
value: 78.1165262000733 |
|
|
- type: cos_sim_recall |
|
|
value: 82.06806282722513 |
|
|
- type: dot_accuracy |
|
|
value: 89.81061047075717 |
|
|
- type: dot_ap |
|
|
value: 87.11746902745236 |
|
|
- type: dot_f1 |
|
|
value: 80.04355498817256 |
|
|
- type: dot_precision |
|
|
value: 78.1165262000733 |
|
|
- type: dot_recall |
|
|
value: 82.06806282722513 |
|
|
- type: euclidean_accuracy |
|
|
value: 89.81061047075717 |
|
|
- type: euclidean_ap |
|
|
value: 87.11746919324248 |
|
|
- type: euclidean_f1 |
|
|
value: 80.04355498817256 |
|
|
- type: euclidean_precision |
|
|
value: 78.1165262000733 |
|
|
- type: euclidean_recall |
|
|
value: 82.06806282722513 |
|
|
- type: manhattan_accuracy |
|
|
value: 89.79508673885202 |
|
|
- type: manhattan_ap |
|
|
value: 87.11074390832218 |
|
|
- type: manhattan_f1 |
|
|
value: 80.13002540726349 |
|
|
- type: manhattan_precision |
|
|
value: 77.83826945412311 |
|
|
- type: manhattan_recall |
|
|
value: 82.56082537727133 |
|
|
- type: max_accuracy |
|
|
value: 89.81061047075717 |
|
|
- type: max_ap |
|
|
value: 87.11747055081017 |
|
|
- type: max_f1 |
|
|
value: 80.13002540726349 |
|
|
language: |
|
|
- multilingual |
|
|
- af |
|
|
- am |
|
|
- ar |
|
|
- as |
|
|
- az |
|
|
- be |
|
|
- bg |
|
|
- bn |
|
|
- br |
|
|
- bs |
|
|
- ca |
|
|
- cs |
|
|
- cy |
|
|
- da |
|
|
- de |
|
|
- el |
|
|
- en |
|
|
- eo |
|
|
- es |
|
|
- et |
|
|
- eu |
|
|
- fa |
|
|
- fi |
|
|
- fr |
|
|
- fy |
|
|
- ga |
|
|
- gd |
|
|
- gl |
|
|
- gu |
|
|
- ha |
|
|
- he |
|
|
- hi |
|
|
- hr |
|
|
- hu |
|
|
- hy |
|
|
- id |
|
|
- is |
|
|
- it |
|
|
- ja |
|
|
- jv |
|
|
- ka |
|
|
- kk |
|
|
- km |
|
|
- kn |
|
|
- ko |
|
|
- ku |
|
|
- ky |
|
|
- la |
|
|
- lo |
|
|
- lt |
|
|
- lv |
|
|
- mg |
|
|
- mk |
|
|
- ml |
|
|
- mn |
|
|
- mr |
|
|
- ms |
|
|
- my |
|
|
- ne |
|
|
- nl |
|
|
- 'no' |
|
|
- om |
|
|
- or |
|
|
- pa |
|
|
- pl |
|
|
- ps |
|
|
- pt |
|
|
- ro |
|
|
- ru |
|
|
- sa |
|
|
- sd |
|
|
- si |
|
|
- sk |
|
|
- sl |
|
|
- so |
|
|
- sq |
|
|
- sr |
|
|
- su |
|
|
- sv |
|
|
- sw |
|
|
- ta |
|
|
- te |
|
|
- th |
|
|
- tl |
|
|
- tr |
|
|
- ug |
|
|
- uk |
|
|
- ur |
|
|
- uz |
|
|
- vi |
|
|
- xh |
|
|
- yi |
|
|
- zh |
|
|
license: mit |
|
|
--- |
|
|
|
|
|
## Multilingual-E5-large-instruct |
|
|
|
|
|
[Multilingual E5 Text Embeddings: A Technical Report](https://arxiv.org/pdf/2402.05672). |
|
|
Liang Wang, Nan Yang, Xiaolong Huang, Linjun Yang, Rangan Majumder, Furu Wei, arXiv 2024 |
|
|
|
|
|
This model has 24 layers and the embedding size is 1024. |
|
|
|
|
|
## Usage |
|
|
|
|
|
Below are examples to encode queries and passages from the MS-MARCO passage ranking dataset. |
|
|
|
|
|
### Transformers |
|
|
|
|
|
```python |
|
|
import torch.nn.functional as F |
|
|
|
|
|
from torch import Tensor |
|
|
from transformers import AutoTokenizer, AutoModel |
|
|
|
|
|
|
|
|
def average_pool(last_hidden_states: Tensor, |
|
|
attention_mask: Tensor) -> Tensor: |
|
|
last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0) |
|
|
return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None] |
|
|
|
|
|
def get_detailed_instruct(task_description: str, query: str) -> str: |
|
|
return f'Instruct: {task_description}\nQuery: {query}' |
|
|
|
|
|
# Each query must come with a one-sentence instruction that describes the task |
|
|
task = 'Given a web search query, retrieve relevant passages that answer the query' |
|
|
queries = [ |
|
|
get_detailed_instruct(task, 'how much protein should a female eat'), |
|
|
get_detailed_instruct(task, '南瓜的家常做法') |
|
|
] |
|
|
# No need to add instruction for retrieval documents |
|
|
documents = [ |
|
|
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", |
|
|
"1.清炒南瓜丝 原料:嫩南瓜半个 调料:葱、盐、白糖、鸡精 做法: 1、南瓜用刀薄薄的削去表面一层皮,用勺子刮去瓤 2、擦成细丝(没有擦菜板就用刀慢慢切成细丝) 3、锅烧热放油,入葱花煸出香味 4、入南瓜丝快速翻炒一分钟左右,放盐、一点白糖和鸡精调味出锅 2.香葱炒南瓜 原料:南瓜1只 调料:香葱、蒜末、橄榄油、盐 做法: 1、将南瓜去皮,切成片 2、油锅8成热后,将蒜末放入爆香 3、爆香后,将南瓜片放入,翻炒 4、在翻炒的同时,可以不时地往锅里加水,但不要太多 5、放入盐,炒匀 6、南瓜差不多软和绵了之后,就可以关火 7、撒入香葱,即可出锅" |
|
|
] |
|
|
input_texts = queries + documents |
|
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained('intfloat/multilingual-e5-large-instruct') |
|
|
model = AutoModel.from_pretrained('intfloat/multilingual-e5-large-instruct') |
|
|
|
|
|
# Tokenize the input texts |
|
|
batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt') |
|
|
|
|
|
outputs = model(**batch_dict) |
|
|
embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask']) |
|
|
|
|
|
# normalize embeddings |
|
|
embeddings = F.normalize(embeddings, p=2, dim=1) |
|
|
scores = (embeddings[:2] @ embeddings[2:].T) * 100 |
|
|
print(scores.tolist()) |
|
|
# => [[91.92852783203125, 67.580322265625], [70.3814468383789, 92.1330795288086]] |
|
|
``` |
|
|
|
|
|
### Sentence Transformers |
|
|
|
|
|
```python |
|
|
# Requires: sentence-transformers |
|
|
from sentence_transformers import SentenceTransformer |
|
|
|
|
|
def get_detailed_instruct(task_description: str, query: str) -> str: |
|
|
return f'Instruct: {task_description}\nQuery: {query}' |
|
|
|
|
|
# Each query must come with a one-sentence instruction that describes the task |
|
|
task = 'Given a web search query, retrieve relevant passages that answer the query' |
|
|
queries = [ |
|
|
get_detailed_instruct(task, 'how much protein should a female eat'), |
|
|
get_detailed_instruct(task, '南瓜的家常做法') |
|
|
] |
|
|
# No need to add instruction for retrieval documents |
|
|
documents = [ |
|
|
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.", |
|
|
"1.清炒南瓜丝 原料:嫩南瓜半个 调料:葱、盐、白糖、鸡精 做法: 1、南瓜用刀薄薄的削去表面一层皮,用勺子刮去瓤 2、擦成细丝(没有擦菜板就用刀慢慢切成细丝) 3、锅烧热放油,入葱花煸出香味 4、入南瓜丝快速翻炒一分钟左右,放盐、一点白糖和鸡精调味出锅 2.香葱炒南瓜 原料:南瓜1只 调料:香葱、蒜末、橄榄油、盐 做法: 1、将南瓜去皮,切成片 2、油锅8成热后,将蒜末放入爆香 3、爆香后,将南瓜片放入,翻炒 4、在翻炒的同时,可以不时地往锅里加水,但不要太多 5、放入盐,炒匀 6、南瓜差不多软和绵了之后,就可以关火 7、撒入香葱,即可出锅" |
|
|
] |
|
|
input_texts = queries + documents |
|
|
|
|
|
model = SentenceTransformer('intfloat/multilingual-e5-large-instruct') |
|
|
|
|
|
embeddings = model.encode(input_texts, convert_to_tensor=True, normalize_embeddings=True) |
|
|
scores = (embeddings[:2] @ embeddings[2:].T) * 100 |
|
|
print(scores.tolist()) |
|
|
# [[91.92853546142578, 67.5802993774414], [70.38143157958984, 92.13307189941406]] |
|
|
``` |
|
|
|
|
|
### Infinity |
|
|
|
|
|
Usage with [Infinity](https://github.com/michaelfeil/infinity): |
|
|
|
|
|
```bash |
|
|
docker run --gpus all -v $PWD/data:/app/.cache -e HF_TOKEN=$HF_TOKEN -p "7997":"7997" \ |
|
|
michaelf34/infinity:0.0.68 \ |
|
|
v2 --model-id intfloat/multilingual-e5-large-instruct --revision "main" --dtype float16 --batch-size 32 --engine torch --port 7997 |
|
|
``` |
|
|
|
|
|
## Supported Languages |
|
|
|
|
|
This model is initialized from [xlm-roberta-large](https://huggingface.co/xlm-roberta-large) |
|
|
and continually trained on a mixture of multilingual datasets. |
|
|
It supports 100 languages from xlm-roberta, |
|
|
but low-resource languages may see performance degradation. |
|
|
|
|
|
## Training Details |
|
|
|
|
|
**Initialization**: [xlm-roberta-large](https://huggingface.co/xlm-roberta-large) |
|
|
|
|
|
**First stage**: contrastive pre-training with 1 billion weakly supervised text pairs. |
|
|
|
|
|
**Second stage**: fine-tuning on datasets from the [E5-mistral](https://arxiv.org/abs/2401.00368) paper. |
|
|
|
|
|
## MTEB Benchmark Evaluation |
|
|
|
|
|
Check out [unilm/e5](https://github.com/microsoft/unilm/tree/master/e5) to reproduce evaluation results |
|
|
on the [BEIR](https://arxiv.org/abs/2104.08663) and [MTEB benchmark](https://arxiv.org/abs/2210.07316). |
|
|
|
|
|
## FAQ |
|
|
|
|
|
**1. Do I need to add instructions to the query?** |
|
|
|
|
|
Yes, this is how the model is trained, otherwise you will see a performance degradation. |
|
|
The task definition should be a one-sentence instruction that describes the task. |
|
|
This is a way to customize text embeddings for different scenarios through natural language instructions. |
|
|
|
|
|
Please check out [unilm/e5/utils.py](https://github.com/microsoft/unilm/blob/9c0f1ff7ca53431fe47d2637dfe253643d94185b/e5/utils.py#L106) for instructions we used for evaluation. |
|
|
|
|
|
On the other hand, there is no need to add instructions to the document side. |
|
|
|
|
|
**2. Why are my reproduced results slightly different from reported in the model card?** |
|
|
|
|
|
Different versions of `transformers` and `pytorch` could cause negligible but non-zero performance differences. |
|
|
|
|
|
**3. Why does the cosine similarity scores distribute around 0.7 to 1.0?** |
|
|
|
|
|
This is a known and expected behavior as we use a low temperature 0.01 for InfoNCE contrastive loss. |
|
|
|
|
|
For text embedding tasks like text retrieval or semantic similarity, |
|
|
what matters is the relative order of the scores instead of the absolute values, |
|
|
so this should not be an issue. |
|
|
|
|
|
## Citation |
|
|
|
|
|
If you find our paper or models helpful, please consider cite as follows: |
|
|
|
|
|
``` |
|
|
@article{wang2024multilingual, |
|
|
title={Multilingual E5 Text Embeddings: A Technical Report}, |
|
|
author={Wang, Liang and Yang, Nan and Huang, Xiaolong and Yang, Linjun and Majumder, Rangan and Wei, Furu}, |
|
|
journal={arXiv preprint arXiv:2402.05672}, |
|
|
year={2024} |
|
|
} |
|
|
``` |
|
|
|
|
|
## Limitations |
|
|
|
|
|
Long texts will be truncated to at most 512 tokens. |
|
|
|
