diff --git a/linker/fine_tune/project.yml b/linker/fine_tune/project.yml old mode 100644 new mode 100755 index 51c6c4f..4f1a6a7 --- a/linker/fine_tune/project.yml +++ b/linker/fine_tune/project.yml @@ -30,11 +30,11 @@ commands: - name: delete_all_files script: - - "python ./project_scripts/delete_all_files.py" + - "python ../../util/fine_tune/delete_all_files.py" - name: delete_all_fine_tunes script: - - "python ./project_scripts/delete_all_fine_tunes.py" + - "python ../../util/fine_tune/delete_all_fine_tunes.py" - name: upload_files deps: @@ -43,20 +43,21 @@ commands: outputs: - 'output/fine_tune_file_ids.json' script: - - 'python ./project_scripts/upload_fine_tune_files.py output/gpt_citation_training.jsonl output/gpt_citation_validation.jsonl' + - 'python ../../util/fine_tune/upload_fine_tune_files.py output/gpt_citation_training.jsonl output/gpt_citation_validation.jsonl' + - name: create_fine_tune deps: - 'output/fine_tune_file_ids.json' script: - - 'python ./project_scripts/create_fine_tune.py ${vars.base_model} ${vars.output_model_suffix}' + - 'python ../../util/fine_tune/create_fine_tune.py ${vars.base_model} ${vars.output_model_suffix}' - name: fine_tune_status script: - - 'python ./project_scripts/fine_tune_status.py' + - 'python ../../util/fine_tune/fine_tune_status.py' - name: fine_tune_stats outputs: - 'output/fine_tune_stats.csv' script: - - './project_scripts/fine_tune_stats.sh ${env.openai_api_key} "file-IJ8nRijlbPpsVJY7eYLwxog9" output/fine_tune_stats.csv' + - '../../util/fine_tune/fine_tune_stats.sh ${env.openai_api_key} "file-IJ8nRijlbPpsVJY7eYLwxog9" output/fine_tune_stats.csv' diff --git a/linker/fine_tune/project_scripts/create_citation_input_for_fine_tuning.py b/linker/fine_tune/project_scripts/create_citation_input_for_fine_tuning.py index af24f2f..9249e47 100644 --- a/linker/fine_tune/project_scripts/create_citation_input_for_fine_tuning.py +++ b/linker/fine_tune/project_scripts/create_citation_input_for_fine_tuning.py @@ -6,7 +6,7 @@ import srsly from util.general import load_mongo_docs from linker.fine_tune.project_scripts import constants -from langchain.chat_models.openai import _convert_message_to_dict +from langchain.chat_models.openai import convert_message_to_dict from langchain.schema import HumanMessage, SystemMessage, AIMessage @@ -86,7 +86,7 @@ def _generate_one_completion_format(doc, is_labeled=False): @staticmethod def serialize_messages(messages): - return {"messages": [_convert_message_to_dict(message) for message in messages]} + return {"messages": [convert_message_to_dict(message) for message in messages]} @staticmethod def _create_system_prompt(): diff --git a/util/fine_tune/__init__.py b/util/fine_tune/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/linker/fine_tune/project_scripts/create_fine_tune.py b/util/fine_tune/create_fine_tune.py similarity index 100% rename from linker/fine_tune/project_scripts/create_fine_tune.py rename to util/fine_tune/create_fine_tune.py diff --git a/linker/fine_tune/project_scripts/delete_all_files.py b/util/fine_tune/delete_all_files.py old mode 100644 new mode 100755 similarity index 100% rename from linker/fine_tune/project_scripts/delete_all_files.py rename to util/fine_tune/delete_all_files.py diff --git a/linker/fine_tune/project_scripts/delete_all_fine_tunes.py b/util/fine_tune/delete_all_fine_tunes.py old mode 100644 new mode 100755 similarity index 100% rename from linker/fine_tune/project_scripts/delete_all_fine_tunes.py rename to util/fine_tune/delete_all_fine_tunes.py diff --git a/linker/fine_tune/project_scripts/fine_tune_stats.sh b/util/fine_tune/fine_tune_stats.sh similarity index 60% rename from linker/fine_tune/project_scripts/fine_tune_stats.sh rename to util/fine_tune/fine_tune_stats.sh index 1123851..840ff2e 100755 --- a/linker/fine_tune/project_scripts/fine_tune_stats.sh +++ b/util/fine_tune/fine_tune_stats.sh @@ -3,14 +3,14 @@ # Check if required inputs are provided if [ $# -ne 3 ]; then echo "Error: Missing required parameters" - echo "Usage: ./fine_tune.sh " + echo "Usage: ./fine_tune.sh " exit 1 fi # Read input variables OPENAI_API_KEY=$1 -FINE_TUNE_JOB_ID=$2 +RESULTS_FILE_ID=$2 OUTPUT_FILE=$3 -curl https://api.openai.com/v1/files/$FINE_TUNE_JOB_ID/content \ +curl https://api.openai.com/v1/files/RESULTS_FILE_ID/content \ -H "Authorization: Bearer $OPENAI_API_KEY" > "${OUTPUT_FILE}" diff --git a/linker/fine_tune/project_scripts/fine_tune_status.py b/util/fine_tune/fine_tune_status.py similarity index 100% rename from linker/fine_tune/project_scripts/fine_tune_status.py rename to util/fine_tune/fine_tune_status.py diff --git a/linker/fine_tune/project_scripts/upload_fine_tune_files.py b/util/fine_tune/upload_fine_tune_files.py similarity index 100% rename from linker/fine_tune/project_scripts/upload_fine_tune_files.py rename to util/fine_tune/upload_fine_tune_files.py