Extend v2 experiments: English bidirectional models, GLUE/MTEB/QA evaluation, and training-free geometry analysis

config/constract_llm/dataset/prepare_english_embedding_data/ft.json

@@ -0,0 +1,44 @@
+{
+  "mode": "ft",
+  "repo_id": "sentence-transformers/embedding-training-data",
+  "revision": "015d73e06b0686135d4d8340b1c68177d204b412",
+  "output_dir": "data/processed/english_embedding/ft",
+  "sources": [
+    {
+      "file_name": "AllNLI.jsonl.gz",
+      "label": "nli",
+      "kind": "triplet"
+    },
+    {
+      "file_name": "msmarco-triplets.jsonl.gz",
+      "label": "msmarco",
+      "kind": "triplet"
+    },
+    {
+      "file_name": "quora_duplicates_triplets.jsonl.gz",
+      "label": "quora",
+      "kind": "triplet"
+    },
+    {
+      "file_name": "NQ-train_pairs.jsonl.gz",
+      "label": "nq",
+      "kind": "pair",
+      "random_negative": true
+    },
+    {
+      "file_name": "squad_pairs.jsonl.gz",
+      "label": "squad",
+      "kind": "pair",
+      "random_negative": true
+    },
+    {
+      "file_name": "TriviaQA_pairs.jsonl.gz",
+      "label": "triviaqa",
+      "kind": "pair",
+      "random_negative": true
+    }
+  ],
+  "validation_fraction": 0.01,
+  "seed": 42,
+  "max_samples_per_source": null
+}

config/constract_llm/dataset/prepare_english_embedding_data/wsl.json

@@ -0,0 +1,137 @@
+{
+  "mode": "wsl",
+  "repo_id": "sentence-transformers/embedding-training-data",
+  "revision": "015d73e06b0686135d4d8340b1c68177d204b412",
+  "output_dir": "data/processed/english_embedding/wsl",
+  "sources": [
+    {
+      "file_name": "NQ-train_pairs.jsonl.gz",
+      "label": "nq",
+      "kind": "pair"
+    },
+    {
+      "file_name": "squad_pairs.jsonl.gz",
+      "label": "squad",
+      "kind": "pair"
+    },
+    {
+      "file_name": "TriviaQA_pairs.jsonl.gz",
+      "label": "triviaqa",
+      "kind": "pair"
+    },
+    {
+      "file_name": "gooaq_pairs.jsonl.gz",
+      "label": "gooaq",
+      "kind": "pair"
+    },
+    {
+      "file_name": "quora_duplicates.jsonl.gz",
+      "label": "quora",
+      "kind": "pair"
+    },
+    {
+      "file_name": "WikiAnswers.jsonl.gz",
+      "label": "wikianswers",
+      "kind": "pair"
+    },
+    {
+      "file_name": "PAQ_pairs.jsonl.gz",
+      "label": "paq",
+      "kind": "pair",
+      "max_samples": 4600000
+    },
+    {
+      "file_name": "amazon-qa.jsonl.gz",
+      "label": "amazon_qa",
+      "kind": "pair"
+    },
+    {
+      "file_name": "eli5_question_answer.jsonl.gz",
+      "label": "eli5",
+      "kind": "pair"
+    },
+    {
+      "file_name": "searchQA_top5_snippets.jsonl.gz",
+      "label": "searchqa",
+      "kind": "pair"
+    },
+    {
+      "file_name": "yahoo_answers_question_answer.jsonl.gz",
+      "label": "yahoo_qa",
+      "kind": "pair"
+    },
+    {
+      "file_name": "yahoo_answers_title_answer.jsonl.gz",
+      "label": "yahoo_title_answer",
+      "kind": "pair"
+    },
+    {
+      "file_name": "yahoo_answers_title_question.jsonl.gz",
+      "label": "yahoo_title_question",
+      "kind": "pair"
+    },
+    {
+      "file_name": "agnews.jsonl.gz",
+      "label": "agnews",
+      "kind": "pair"
+    },
+    {
+      "file_name": "ccnews_title_text.jsonl.gz",
+      "label": "ccnews",
+      "kind": "pair"
+    },
+    {
+      "file_name": "npr.jsonl.gz",
+      "label": "npr",
+      "kind": "pair"
+    },
+    {
+      "file_name": "cnn_dailymail.jsonl.gz",
+      "label": "cnn_dailymail",
+      "kind": "pair"
+    },
+    {
+      "file_name": "xsum.jsonl.gz",
+      "label": "xsum",
+      "kind": "pair"
+    },
+    {
+      "file_name": "stackexchange_duplicate_questions_title_title.jsonl.gz",
+      "label": "stackexchange_title",
+      "kind": "pair"
+    },
+    {
+      "file_name": "stackexchange_duplicate_questions_body_body.jsonl.gz",
+      "label": "stackexchange_body",
+      "kind": "pair"
+    },
+    {
+      "file_name": "stackexchange_duplicate_questions_title-body_title-body.jsonl.gz",
+      "label": "stackexchange_title_body",
+      "kind": "pair"
+    },
+    {
+      "file_name": "altlex.jsonl.gz",
+      "label": "altlex",
+      "kind": "pair"
+    },
+    {
+      "file_name": "SimpleWiki.jsonl.gz",
+      "label": "simplewiki",
+      "kind": "pair"
+    },
+    {
+      "file_name": "sentence-compression.jsonl.gz",
+      "label": "sentence_compression",
+      "kind": "pair"
+    },
+    {
+      "file_name": "wikihow.jsonl.gz",
+      "label": "wikihow",
+      "kind": "pair"
+    }
+  ],
+  "validation_fraction": 0.01,
+  "seed": 42,
+  "max_samples_per_source": null
+}

config/constract_llm/eval/analysis/example.json

@@ -0,0 +1,18 @@
+{
+    "model_name_or_path": "iamtatsuki05/Sentence-ModernBERT-JP-0.5B",
+    "model_revision": null,
+    "output_dir": "data/outputs/eval/analysis/Sentence-ModernBERT-JP-0.5B",
+    "language": "ja",
+    "num_examples": 2000,
+    "num_attention_examples": 32,
+    "attention_max_tokens": 128,
+    "batch_size": 16,
+    "max_seq_length": 512,
+    "seed": 42,
+    "dtype": "bfloat16",
+    "with_attention": true,
+    "with_layer_sts": true,
+    "with_kill_test": true,
+    "dump_layer_embeddings": false,
+    "sts_max_pairs": 1500
+}

config/constract_llm/eval/bench_sbert/sentence-instructlm-bi-en-0.5b.json

@@ -0,0 +1,35 @@
+{
+  "model_name_or_path": "iamtatsuki05/Sentence-InstructLM-Bi-EN-0.5B",
+  "output_dir": "data/outputs/bench_sbert/sentence-instructlm-bi-en-0.5b",
+  "dataset_name_or_path": "google/wiki40b",
+  "dataset_config_name": "en",
+  "dataset_split": "train",
+  "text_fields": [
+    "text"
+  ],
+  "streaming": false,
+  "batch_sizes": [
+    16,
+    32,
+    64,
+    128,
+    256
+  ],
+  "seq_lengths": [
+    128,
+    256,
+    512
+  ],
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "normalize_embeddings": true,
+  "max_use_samples": 100000,
+  "monitor_interval_ms": 100,
+  "warmup_fraction": 0.0,
+  "show_progress": true,
+  "trust_remote_code": true,
+  "attn_implementation": "flash_attention_2",
+  "model_extra_kwargs": {},
+  "tokenizer_extra_kwargs": {},
+  "common_extra_kwargs": {}
+}

config/constract_llm/eval/bench_sbert/sentence-llama-bi-1b.json

@@ -0,0 +1,35 @@
+{
+  "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-1B",
+  "output_dir": "data/outputs/bench_sbert/llama-bi-1b",
+  "dataset_name_or_path": "fujiki/wiki40b_ja",
+  "dataset_config_name": "default",
+  "dataset_split": "train",
+  "text_fields": [
+    "text"
+  ],
+  "streaming": true,
+  "batch_sizes": [
+    16,
+    32,
+    64,
+    128,
+    256
+  ],
+  "seq_lengths": [
+    128,
+    256,
+    512
+  ],
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "normalize_embeddings": true,
+  "max_use_samples": 100000,
+  "monitor_interval_ms": 100,
+  "warmup_fraction": 0.0,
+  "show_progress": true,
+  "trust_remote_code": true,
+  "attn_implementation": "flash_attention_2",
+  "model_extra_kwargs": {},
+  "tokenizer_extra_kwargs": {},
+  "common_extra_kwargs": {}
+}

config/constract_llm/eval/bench_sbert/sentence-llama-bi-3b.json

@@ -0,0 +1,35 @@
+{
+    "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-JP-3B",
+    "output_dir": "data/outputs/bench_sbert/llama-bi-3b",
+    "dataset_name_or_path": "fujiki/wiki40b_ja",
+    "dataset_config_name": "default",
+    "dataset_split": "train",
+    "text_fields": [
+        "text"
+    ],
+    "streaming": true,
+    "batch_sizes": [
+        16,
+        32,
+        64,
+        128,
+        256
+    ],
+    "seq_lengths": [
+        128,
+        256,
+        512
+    ],
+    "device": "cuda",
+    "dtype": "bfloat16",
+    "normalize_embeddings": true,
+    "max_use_samples": 100000,
+    "monitor_interval_ms": 100,
+    "warmup_fraction": 0.0,
+    "show_progress": true,
+    "trust_remote_code": true,
+    "attn_implementation": "flash_attention_2",
+    "model_extra_kwargs": {},
+    "tokenizer_extra_kwargs": {},
+    "common_extra_kwargs": {}
+}

config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-0.5b.json

@@ -0,0 +1,35 @@
+{
+  "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-0.5B",
+  "output_dir": "data/outputs/bench_sbert/sentence-llama-bi-en-0.5b",
+  "dataset_name_or_path": "google/wiki40b",
+  "dataset_config_name": "en",
+  "dataset_split": "train",
+  "text_fields": [
+    "text"
+  ],
+  "streaming": true,
+  "batch_sizes": [
+    16,
+    32,
+    64,
+    128,
+    256
+  ],
+  "seq_lengths": [
+    128,
+    256,
+    512
+  ],
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "normalize_embeddings": true,
+  "max_use_samples": 100000,
+  "monitor_interval_ms": 100,
+  "warmup_fraction": 0.0,
+  "show_progress": true,
+  "trust_remote_code": true,
+  "attn_implementation": "flash_attention_2",
+  "model_extra_kwargs": {},
+  "tokenizer_extra_kwargs": {},
+  "common_extra_kwargs": {}
+}

config/constract_llm/eval/bench_sbert/sentence-llama-bi-en-1b.json

@@ -0,0 +1,35 @@
+{
+  "model_name_or_path": "iamtatsuki05/Sentence-Llama-Bi-EN-1B",
+  "output_dir": "data/outputs/bench_sbert/sentence-llama-bi-en-1b",
+  "dataset_name_or_path": "google/wiki40b",
+  "dataset_config_name": "en",
+  "dataset_split": "train",
+  "text_fields": [
+    "text"
+  ],
+  "streaming": true,
+  "batch_sizes": [
+    16,
+    32,
+    64,
+    128,
+    256
+  ],
+  "seq_lengths": [
+    128,
+    256,
+    512
+  ],
+  "device": "cuda",
+  "dtype": "bfloat16",
+  "normalize_embeddings": true,
+  "max_use_samples": 100000,
+  "monitor_interval_ms": 100,
+  "warmup_fraction": 0.0,
+  "show_progress": true,
+  "trust_remote_code": true,
+  "attn_implementation": "flash_attention_2",
+  "model_extra_kwargs": {},
+  "tokenizer_extra_kwargs": {},
+  "common_extra_kwargs": {}
+}