Migrate to uv package management

README.md

@@ -8,9 +8,7 @@
 ## 仮想環境
 
 ```bash
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requrements.txt
+uv sync
 ```
 
 ## データセット
@@ -66,15 +64,15 @@ MODEL_NAME=llm-jp/llm-jp-3-1.8b-instruct
 OUTPUT_DIR=./output/llm-jp-3-1.8b-instruct
 
 # 生成
-python3 -m src.llm_jp_judge.generate \
+uv run python3 -m src.llm_jp_judge.generate \
     output.dir=$OUTPUT_DIR/generation \
     client=vllm \
     client.model_name=$MODEL_NAME \
     benchmark.quality.dataset.path=./data/cache/llm-jp/llm-jp-instructions/v1.0/test.json \
     benchmark.safety.dataset.path=./data/cache/llm-jp/AnswerCarefully/v2.0/test.json
 
 # 評価
-python3 -m src.llm_jp_judge.evaluate \
+uv run python3 -m src.llm_jp_judge.evaluate \
     input.dir=$OUTPUT_DIR/generation \
     output.dir=$OUTPUT_DIR/evaluation \
     client=azure \
@@ -137,7 +135,7 @@ python3 -m src.llm_jp_judge.evaluate \
 非アクティブにしたい場合は、生成時に以下のように指定してください。
 
 ```
-python3 -m src.llm_jp_judge.generate \
+uv run python3 -m src.llm_jp_judge.generate \
     benchmark.mt_bench.dataset.path=null
 ```
 
@@ -149,7 +147,7 @@ python3 -m src.llm_jp_judge.generate \
 非アクティブにしたい場合は、生成時に以下のように指定してください。
 
 ```
-python3 -m src.llm_jp_judge.generate \
+uv run python3 -m src.llm_jp_judge.generate \
     benchmark.mt_bench.dataset.path=null
 ```
 
@@ -162,7 +160,7 @@ python3 -m src.llm_jp_judge.generate \
 OpenAI API のモデル名(例:`gpt-4o-2024-08-06`)を指定できます。
 
 ```
-python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
+uv run python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
     client=openai \
     client.model_name=gpt-4o-2024-08-06 \  # モデル名
     client.async_request_interval=0.5  # APIリクエストの間隔(秒)
@@ -176,7 +174,7 @@ python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
 Azure OpenAI APIのデプロイ名(例:`gpt-4o-2024-08-06`)を指定できます。
 
 ```
-python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
+uv run python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
     client=azure \
     client.model_name=gpt-4o-2024-08-06 \  # デプロイ名
     client.async_request_interval=0.5  # APIリクエストの間隔(秒)
@@ -187,7 +185,7 @@ python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
 AWS Bedrock APIのデプロイ名(例:`anthropic.claude-3-5-sonnet-20240620-v1:0`)を指定できます。  
 
 ```
-python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
+uv run python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
     client=bedrock \
     client.model_name=anthropic.claude-3-5-sonnet-20240620-v1:0 \  # デプロイ名
     client.async_request_interval=10  # APIリクエストの間隔(秒)
@@ -202,7 +200,7 @@ Hugging Faceのモデル名(例:`llm-jp/llm-jp-3-1.8b-instruct`)もしくはパ
 > 対応していない場合、チャットテンプレートに対応したトークナイザーを`client.tokenizer_name`として指定するか、jinja形式のチャットテンプレートを`client.chat_template.path`として与えてください。
 
 ```bash
-python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
+uv run python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
     client=vllm \
     client.model_name=llm-jp/llm-jp-3-1.8b-instruct # Huggin Faceのモデル名 or パス
 ```
@@ -217,7 +215,7 @@ python3 -m src.llm_jp_judge.evaluate \ # generate or evaluate
 `{entity_name}`、`{project_name}`、`{run_name}`は適宜設定してください。
 
 ```
-python3 -m src.llm_jp_judge.evaluate \
+uv run python3 -m src.llm_jp_judge.evaluate \
     dashboard=wandb \
     dashboard.entity={entity_name} \
     dashboard.project={project_name} \

pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "llm-jp-judge"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "anthropic==0.49.0",
+    "awscli==1.38.5",
+    "boto3==1.37.5",
+    "datasets>=4.1.1",
+    "hydra-core==1.3.2",
+    "openai==1.65.2",
+    "python-dotenv==1.0.1",
+    "transformers<4.54.0",
+    "vllm>=0.6.2",
+    "wandb==0.19.7",
+]

requrements.txt

@@ -1,8 +1,9 @@
+anthropic==0.49.0
+awscli==1.38.5
+boto3==1.37.5
 hydra-core==1.3.2
 openai==1.65.2
-vllm==0.6.2
-anthropic==0.49.0
 python-dotenv==1.0.1
-wandb==0.19.7
-awscli==1.38.5
-boto3==1.37.5
+transformers<4.54.0
+vllm>=0.6.2
+wandb==0.19.7

src/llm_jp_judge/__main__.py

@@ -1,6 +1,5 @@
 import hydra
 
-
 from . import generate
 
 

src/llm_jp_judge/client/__init__.py

@@ -1,5 +1,5 @@
-from .remote import OpenAI, AzureOpenAI, BedrockAnthropic
 from .local import vLLMClient
+from .remote import AzureOpenAI, BedrockAnthropic, OpenAI
 
 
 def load_client(name="azure", **kwargs):

src/llm_jp_judge/client/local.py

@@ -2,10 +2,9 @@
 from copy import deepcopy
 
 import hydra
-
 import torch
-from vllm import LLM, SamplingParams
 from huggingface_hub import repo_exists
+from vllm import LLM, SamplingParams
 
 NUM_GPUS = torch.cuda.device_count()
 

src/llm_jp_judge/client/remote.py

@@ -1,17 +1,15 @@
-import os
+import asyncio
 import logging
+import os
 import warnings
-import asyncio
 
 import openai
-from openai import OpenAI as OpenAIClient
-from openai import AzureOpenAI as AzureOpenAIClient
-from anthropic import AnthropicBedrock as AnthropicBedrockClient
-
-from dotenv import load_dotenv
-
 import tqdm
 import tqdm.asyncio
+from anthropic import AnthropicBedrock as AnthropicBedrockClient
+from dotenv import load_dotenv
+from openai import AzureOpenAI as AzureOpenAIClient
+from openai import OpenAI as OpenAIClient
 
 from .local import BaseClient
 

src/llm_jp_judge/dashboard/__init__.py

@@ -1,5 +1,5 @@
-from .wandb import WandB
 from .base import BaseDashboard
+from .wandb import WandB
 
 
 def load_dashboard(cfg, name=None, **kwargs):

src/llm_jp_judge/dashboard/base.py

@@ -1,5 +1,5 @@
-import os
 import json
+import os
 
 
 class BaseDashboard:

src/llm_jp_judge/dashboard/wandb.py

@@ -1,6 +1,5 @@
-import wandb
-
 import omegaconf
+import wandb
 
 from .base import BaseDashboard
 

src/llm_jp_judge/dataset/__init__.py

@@ -1,6 +1,6 @@
-from .safety import load_safety
-from .quality import load_quality
 from .mt_bench import load_mt_bench
+from .quality import load_quality
+from .safety import load_safety
 
 
 def load_dataset(name, path, size=None):

src/llm_jp_judge/dataset/mt_bench.py

@@ -1,4 +1,5 @@
 import json
+
 import hydra
 
 

src/llm_jp_judge/dataset/quality.py

@@ -1,4 +1,5 @@
 import json
+
 import hydra
 
 

src/llm_jp_judge/dataset/safety.py

@@ -1,4 +1,5 @@
 import json
+
 import hydra
 
 

src/llm_jp_judge/evaluate.py

@@ -1,14 +1,14 @@
-import re
-import os
 import glob
+import logging
+import os
+import re
 
 import hydra
-import logging
 
 from .client import load_client
-from .evaluator import load_evaluator
-from .utils.data import load_jsonl, load_json
 from .dashboard import load_dashboard
+from .evaluator import load_evaluator
+from .utils.data import load_json, load_jsonl
 
 
 def load_metadata(cfg):

src/llm_jp_judge/evaluator/__init__.py

@@ -1,6 +1,6 @@
+from .mt_bench import MTBenchEvaluator
 from .quality import QualityEvaluator
 from .safety import SafetyEvaluator
-from .mt_bench import MTBenchEvaluator
 
 
 def load_evaluator(client, dashboard, metadata={}, metric="abs_quality", **kwargs):

src/llm_jp_judge/evaluator/base.py

@@ -1,6 +1,6 @@
-import re
 import json
 import logging
+import re
 
 
 class BaseScoreExtractor(object):

src/llm_jp_judge/evaluator/mt_bench.py

@@ -1,11 +1,10 @@
-import logging
-
 import json
-from copy import deepcopy
+import logging
 from collections import defaultdict
+from copy import deepcopy
 
-from .base import BaseEvaluator, BaseScoreExtractor
 from ..utils.data import load_jsonl
+from .base import BaseEvaluator, BaseScoreExtractor
 
 
 class MTBenchEvaluator(BaseEvaluator):

src/llm_jp_judge/evaluator/quality.py

@@ -1,9 +1,8 @@
-import re
 import json
 import logging
-
-from copy import deepcopy
+import re
 from collections import defaultdict
+from copy import deepcopy
 
 from .base import BaseEvaluator
 

src/llm_jp_judge/evaluator/safety.py

@@ -1,8 +1,7 @@
-import re
 import logging
-
-from copy import deepcopy
+import re
 from collections import defaultdict
+from copy import deepcopy
 
 from .base import BaseEvaluator, BaseScoreExtractor
 

src/llm_jp_judge/generate.py

@@ -1,15 +1,13 @@
+import logging
 import os
-import hydra
-
 from collections import defaultdict
 
+import hydra
 from omegaconf import OmegaConf
 
 from .client import load_client
-from .utils.data import save_jsonl, save_json
 from .dataset import load_dataset
-
-import logging
+from .utils.data import save_json, save_jsonl
 
 
 def generate(cfg, client, benchmark_cfg):

src/llm_jp_judge/utils/data.py

@@ -1,5 +1,5 @@
-import os
 import json
+import os
 
 import hydra