metadata
tags:
- mteb
model-index:
- name: conan-embedding
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 57.32391831434286
- type: cos_sim_spearman
value: 60.95420518306528
- type: euclidean_pearson
value: 58.73713689471779
- type: euclidean_spearman
value: 60.05871977323687
- type: manhattan_pearson
value: 58.71439394187201
- type: manhattan_spearman
value: 60.03726849511567
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 57.416459629680325
- type: cos_sim_spearman
value: 58.78935983944373
- type: euclidean_pearson
value: 62.569916488206054
- type: euclidean_spearman
value: 58.32089170859326
- type: manhattan_pearson
value: 62.552144365725816
- type: manhattan_spearman
value: 58.31304102953674
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 50.364
- type: f1
value: 47.373487235615706
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 73.16013393385914
- type: cos_sim_spearman
value: 74.79454418123198
- type: euclidean_pearson
value: 72.91991570850215
- type: euclidean_spearman
value: 74.40420227973465
- type: manhattan_pearson
value: 72.91482392990748
- type: manhattan_spearman
value: 74.40097720245406
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 59.86170547809245
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 50.38135526839833
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 90.92142640302838
- type: mrr
value: 92.76190476190476
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 90.0539331525924
- type: mrr
value: 92.00964285714285
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.480999999999998
- type: map_at_10
value: 40.129
- type: map_at_100
value: 42.025
- type: map_at_1000
value: 42.123
- type: map_at_3
value: 35.644
- type: map_at_5
value: 38.187
- type: mrr_at_1
value: 40.01
- type: mrr_at_10
value: 48.886
- type: mrr_at_100
value: 49.825
- type: mrr_at_1000
value: 49.864000000000004
- type: mrr_at_3
value: 46.178000000000004
- type: mrr_at_5
value: 47.711999999999996
- type: ndcg_at_1
value: 40.01
- type: ndcg_at_10
value: 47.032000000000004
- type: ndcg_at_100
value: 54.135
- type: ndcg_at_1000
value: 55.821
- type: ndcg_at_3
value: 41.377
- type: ndcg_at_5
value: 43.808
- type: precision_at_1
value: 40.01
- type: precision_at_10
value: 10.495000000000001
- type: precision_at_100
value: 1.628
- type: precision_at_1000
value: 0.184
- type: precision_at_3
value: 23.648
- type: precision_at_5
value: 17.224
- type: recall_at_1
value: 26.480999999999998
- type: recall_at_10
value: 58.557
- type: recall_at_100
value: 87.52799999999999
- type: recall_at_1000
value: 98.80600000000001
- type: recall_at_3
value: 41.628
- type: recall_at_5
value: 49.013
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 86.5423932651834
- type: cos_sim_ap
value: 92.8551948361576
- type: cos_sim_f1
value: 87.07992733878292
- type: cos_sim_precision
value: 84.6391525049658
- type: cos_sim_recall
value: 89.66565349544074
- type: dot_accuracy
value: 77.05351773902585
- type: dot_ap
value: 85.41568844524294
- type: dot_f1
value: 79.17229905375896
- type: dot_precision
value: 71.29213483146067
- type: dot_recall
value: 89.01098901098901
- type: euclidean_accuracy
value: 84.49789536981359
- type: euclidean_ap
value: 91.54179322199462
- type: euclidean_f1
value: 85.58362369337979
- type: euclidean_precision
value: 80.08966782147951
- type: euclidean_recall
value: 91.88683656768764
- type: manhattan_accuracy
value: 84.41371016235718
- type: manhattan_ap
value: 91.5209564727476
- type: manhattan_f1
value: 85.5386606276286
- type: manhattan_precision
value: 79.38350680544436
- type: manhattan_recall
value: 92.72854804769698
- type: max_accuracy
value: 86.5423932651834
- type: max_ap
value: 92.8551948361576
- type: max_f1
value: 87.07992733878292
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 82.824
- type: map_at_10
value: 89.749
- type: map_at_100
value: 89.79899999999999
- type: map_at_1000
value: 89.8
- type: map_at_3
value: 89.18599999999999
- type: map_at_5
value: 89.586
- type: mrr_at_1
value: 83.035
- type: mrr_at_10
value: 89.699
- type: mrr_at_100
value: 89.749
- type: mrr_at_1000
value: 89.749
- type: mrr_at_3
value: 89.18199999999999
- type: mrr_at_5
value: 89.582
- type: ndcg_at_1
value: 83.14
- type: ndcg_at_10
value: 92.059
- type: ndcg_at_100
value: 92.292
- type: ndcg_at_1000
value: 92.304
- type: ndcg_at_3
value: 91.033
- type: ndcg_at_5
value: 91.716
- type: precision_at_1
value: 83.14
- type: precision_at_10
value: 9.989
- type: precision_at_100
value: 1.009
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 32.35
- type: precision_at_5
value: 19.747
- type: recall_at_1
value: 82.824
- type: recall_at_10
value: 98.84100000000001
- type: recall_at_100
value: 99.895
- type: recall_at_1000
value: 100
- type: recall_at_3
value: 96.207
- type: recall_at_5
value: 97.84
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.839000000000002
- type: map_at_10
value: 81.363
- type: map_at_100
value: 84.265
- type: map_at_1000
value: 84.29299999999999
- type: map_at_3
value: 56.593
- type: map_at_5
value: 71.057
- type: mrr_at_1
value: 91.14999999999999
- type: mrr_at_10
value: 94.00800000000001
- type: mrr_at_100
value: 94.059
- type: mrr_at_1000
value: 94.06
- type: mrr_at_3
value: 93.692
- type: mrr_at_5
value: 93.874
- type: ndcg_at_1
value: 91.14999999999999
- type: ndcg_at_10
value: 88.584
- type: ndcg_at_100
value: 91.186
- type: ndcg_at_1000
value: 91.437
- type: ndcg_at_3
value: 87.287
- type: ndcg_at_5
value: 86.058
- type: precision_at_1
value: 91.14999999999999
- type: precision_at_10
value: 42.199999999999996
- type: precision_at_100
value: 4.845
- type: precision_at_1000
value: 0.49
- type: precision_at_3
value: 78.05
- type: precision_at_5
value: 65.53
- type: recall_at_1
value: 26.839000000000002
- type: recall_at_10
value: 89.91900000000001
- type: recall_at_100
value: 98.18900000000001
- type: recall_at_1000
value: 99.503
- type: recall_at_3
value: 58.622
- type: recall_at_5
value: 75.44
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 55.300000000000004
- type: map_at_10
value: 65.53
- type: map_at_100
value: 66.084
- type: map_at_1000
value: 66.09
- type: map_at_3
value: 62.9
- type: map_at_5
value: 64.45
- type: mrr_at_1
value: 55.300000000000004
- type: mrr_at_10
value: 65.53
- type: mrr_at_100
value: 66.084
- type: mrr_at_1000
value: 66.09
- type: mrr_at_3
value: 62.9
- type: mrr_at_5
value: 64.45
- type: ndcg_at_1
value: 55.300000000000004
- type: ndcg_at_10
value: 70.743
- type: ndcg_at_100
value: 73.202
- type: ndcg_at_1000
value: 73.379
- type: ndcg_at_3
value: 65.366
- type: ndcg_at_5
value: 68.142
- type: precision_at_1
value: 55.300000000000004
- type: precision_at_10
value: 8.72
- type: precision_at_100
value: 0.9820000000000001
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 24.166999999999998
- type: precision_at_5
value: 15.840000000000002
- type: recall_at_1
value: 55.300000000000004
- type: recall_at_10
value: 87.2
- type: recall_at_100
value: 98.2
- type: recall_at_1000
value: 99.6
- type: recall_at_3
value: 72.5
- type: recall_at_5
value: 79.2
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 51.82762601000385
- type: f1
value: 39.89843169307487
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 89.13696060037525
- type: ap
value: 60.815127909851284
- type: f1
value: 84.4053710993565
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 74.49480212942174
- type: cos_sim_spearman
value: 79.79417204577828
- type: euclidean_pearson
value: 79.53588770578706
- type: euclidean_spearman
value: 79.44601707954529
- type: manhattan_pearson
value: 79.52732262295254
- type: manhattan_spearman
value: 79.43565470474867
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 41.45350792019148
- type: mrr
value: 41.2468253968254
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 68.327
- type: map_at_10
value: 78.244
- type: map_at_100
value: 78.493
- type: map_at_1000
value: 78.498
- type: map_at_3
value: 76.305
- type: map_at_5
value: 77.549
- type: mrr_at_1
value: 70.63000000000001
- type: mrr_at_10
value: 78.78399999999999
- type: mrr_at_100
value: 79.001
- type: mrr_at_1000
value: 79.00500000000001
- type: mrr_at_3
value: 77.11099999999999
- type: mrr_at_5
value: 78.175
- type: ndcg_at_1
value: 70.63000000000001
- type: ndcg_at_10
value: 82.221
- type: ndcg_at_100
value: 83.281
- type: ndcg_at_1000
value: 83.403
- type: ndcg_at_3
value: 78.56400000000001
- type: ndcg_at_5
value: 80.65299999999999
- type: precision_at_1
value: 70.63000000000001
- type: precision_at_10
value: 9.983
- type: precision_at_100
value: 1.05
- type: precision_at_1000
value: 0.106
- type: precision_at_3
value: 29.69
- type: precision_at_5
value: 18.931
- type: recall_at_1
value: 68.327
- type: recall_at_10
value: 93.91000000000001
- type: recall_at_100
value: 98.56
- type: recall_at_1000
value: 99.508
- type: recall_at_3
value: 84.262
- type: recall_at_5
value: 89.21
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 78.34566240753193
- type: f1
value: 74.84529699027114
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 86.35171486213852
- type: f1
value: 85.47178380961844
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 57.199999999999996
- type: map_at_10
value: 65.075
- type: map_at_100
value: 65.607
- type: map_at_1000
value: 65.63
- type: map_at_3
value: 63
- type: map_at_5
value: 64.145
- type: mrr_at_1
value: 57.099999999999994
- type: mrr_at_10
value: 65.024
- type: mrr_at_100
value: 65.556
- type: mrr_at_1000
value: 65.579
- type: mrr_at_3
value: 62.949999999999996
- type: mrr_at_5
value: 64.095
- type: ndcg_at_1
value: 57.199999999999996
- type: ndcg_at_10
value: 69.083
- type: ndcg_at_100
value: 71.844
- type: ndcg_at_1000
value: 72.41499999999999
- type: ndcg_at_3
value: 64.781
- type: ndcg_at_5
value: 66.842
- type: precision_at_1
value: 57.199999999999996
- type: precision_at_10
value: 8.18
- type: precision_at_100
value: 0.951
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 23.3
- type: precision_at_5
value: 14.979999999999999
- type: recall_at_1
value: 57.199999999999996
- type: recall_at_10
value: 81.8
- type: recall_at_100
value: 95.1
- type: recall_at_1000
value: 99.5
- type: recall_at_3
value: 69.89999999999999
- type: recall_at_5
value: 74.9
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 79.10000000000001
- type: f1
value: 78.84641270838914
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 85.76069301570114
- type: cos_sim_ap
value: 91.2632425363724
- type: cos_sim_f1
value: 86.8038133467135
- type: cos_sim_precision
value: 82.69598470363289
- type: cos_sim_recall
value: 91.34107708553326
- type: dot_accuracy
value: 79.48023822414727
- type: dot_ap
value: 86.96279505479045
- type: dot_f1
value: 81.3658536585366
- type: dot_precision
value: 75.61196736174071
- type: dot_recall
value: 88.0675818373812
- type: euclidean_accuracy
value: 84.0281537628587
- type: euclidean_ap
value: 88.51741173181273
- type: euclidean_f1
value: 85.51791850760924
- type: euclidean_precision
value: 79.90825688073394
- type: euclidean_recall
value: 91.97465681098205
- type: manhattan_accuracy
value: 84.08229561451002
- type: manhattan_ap
value: 88.47110130415778
- type: manhattan_f1
value: 85.5886663409868
- type: manhattan_precision
value: 79.63636363636364
- type: manhattan_recall
value: 92.5026399155227
- type: max_accuracy
value: 85.76069301570114
- type: max_ap
value: 91.2632425363724
- type: max_f1
value: 86.8038133467135
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 95.24999999999999
- type: ap
value: 93.62998298041074
- type: f1
value: 95.24017532648074
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 42.4435049928267
- type: cos_sim_spearman
value: 48.30517824065838
- type: euclidean_pearson
value: 47.06361699313179
- type: euclidean_spearman
value: 47.82186765650415
- type: manhattan_pearson
value: 47.07696683801967
- type: manhattan_spearman
value: 47.8382411727149
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 42.98576550573372
- type: cos_sim_spearman
value: 45.186068114717166
- type: euclidean_pearson
value: 34.35887865584346
- type: euclidean_spearman
value: 40.16452917420738
- type: manhattan_pearson
value: 34.32064302728564
- type: manhattan_spearman
value: 40.14426009784696
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 67.18376994296008
- type: cos_sim_spearman
value: 68.25525695381309
- type: euclidean_pearson
value: 65.10112612702181
- type: euclidean_spearman
value: 67.3953850420758
- type: manhattan_pearson
value: 64.96220731298227
- type: manhattan_spearman
value: 67.35995395052407
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 80.23573559451846
- type: cos_sim_spearman
value: 81.88824265805962
- type: euclidean_pearson
value: 80.25198665320175
- type: euclidean_spearman
value: 80.78767800619003
- type: manhattan_pearson
value: 80.25183889871069
- type: manhattan_spearman
value: 80.77487518316391
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 69.0943636065547
- type: mrr
value: 79.81629026017988
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 27.338
- type: map_at_10
value: 76.943
- type: map_at_100
value: 80.632
- type: map_at_1000
value: 80.695
- type: map_at_3
value: 53.946000000000005
- type: map_at_5
value: 66.399
- type: mrr_at_1
value: 89.47
- type: mrr_at_10
value: 92.31200000000001
- type: mrr_at_100
value: 92.406
- type: mrr_at_1000
value: 92.40899999999999
- type: mrr_at_3
value: 91.89
- type: mrr_at_5
value: 92.167
- type: ndcg_at_1
value: 89.47
- type: ndcg_at_10
value: 84.629
- type: ndcg_at_100
value: 88.278
- type: ndcg_at_1000
value: 88.871
- type: ndcg_at_3
value: 85.80600000000001
- type: ndcg_at_5
value: 84.531
- type: precision_at_1
value: 89.47
- type: precision_at_10
value: 42.077999999999996
- type: precision_at_100
value: 5.023
- type: precision_at_1000
value: 0.516
- type: precision_at_3
value: 75.016
- type: precision_at_5
value: 62.980000000000004
- type: recall_at_1
value: 27.338
- type: recall_at_10
value: 83.889
- type: recall_at_100
value: 95.674
- type: recall_at_1000
value: 98.65
- type: recall_at_3
value: 55.85099999999999
- type: recall_at_5
value: 70.131
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 54.70900000000001
- type: f1
value: 52.74258250140307
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 77.01405081546925
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 71.50626450459885
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 64.1
- type: map_at_10
value: 75.047
- type: map_at_100
value: 75.347
- type: map_at_1000
value: 75.348
- type: map_at_3
value: 73.333
- type: map_at_5
value: 74.313
- type: mrr_at_1
value: 64.1
- type: mrr_at_10
value: 75.047
- type: mrr_at_100
value: 75.347
- type: mrr_at_1000
value: 75.348
- type: mrr_at_3
value: 73.333
- type: mrr_at_5
value: 74.313
- type: ndcg_at_1
value: 64.1
- type: ndcg_at_10
value: 79.814
- type: ndcg_at_100
value: 81.071
- type: ndcg_at_1000
value: 81.085
- type: ndcg_at_3
value: 76.307
- type: ndcg_at_5
value: 78.054
- type: precision_at_1
value: 64.1
- type: precision_at_10
value: 9.45
- type: precision_at_100
value: 0.9990000000000001
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 28.299999999999997
- type: precision_at_5
value: 17.82
- type: recall_at_1
value: 64.1
- type: recall_at_10
value: 94.5
- type: recall_at_100
value: 99.9
- type: recall_at_1000
value: 100
- type: recall_at_3
value: 84.89999999999999
- type: recall_at_5
value: 89.1
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 89.61
- type: ap
value: 75.72595764105405
- type: f1
value: 88.23268907984898
license: cc-by-nc-4.0
Conan-embedding-v1
Performance
| Model | Average | CLS | Clustering | Reranking | Retrieval | STS | Pair_CLS |
|---|---|---|---|---|---|---|---|
| gte-Qwen2-7B-instruct | 72.05 | 75.09 | 66.06 | 68.92 | 76.03 | 65.33 | 87.48 |
| xiaobu-embedding-v2 | 72.43 | 74.67 | 65.17 | 72.58 | 76.5 | 64.53 | 91.87 |
| Conan-embedding-v1 | 72.61 | 74.97 | 64.69 | 72.88 | 76.77 | 64.75 | 92.06 |
More details will be available soon.
About
Created by the Tencent BAC Group. All rights reserved.
License
This work is licensed under a Creative Commons Attribution-NonCommercial 4.0 International License.