chore: re-arrange repo such that useful scripts can be shared by many…

linker/fine_tune/project.yml

@@ -30,11 +30,11 @@ commands:
 
   - name: delete_all_files
     script:
-      - "python ./project_scripts/delete_all_files.py"
+      - "python ../../util/fine_tune/delete_all_files.py"
 
   - name: delete_all_fine_tunes
     script:
-      - "python ./project_scripts/delete_all_fine_tunes.py"
+      - "python ../../util/fine_tune/delete_all_fine_tunes.py"
 
   - name: upload_files
     deps:
@@ -43,20 +43,21 @@ commands:
     outputs:
       - 'output/fine_tune_file_ids.json'
     script:
-      - 'python ./project_scripts/upload_fine_tune_files.py output/gpt_citation_training.jsonl output/gpt_citation_validation.jsonl'
+      - 'python ../../util/fine_tune/upload_fine_tune_files.py output/gpt_citation_training.jsonl output/gpt_citation_validation.jsonl'
+
 
   - name: create_fine_tune
     deps:
       - 'output/fine_tune_file_ids.json'
     script:
-      - 'python ./project_scripts/create_fine_tune.py ${vars.base_model} ${vars.output_model_suffix}'
+      - 'python ../../util/fine_tune/create_fine_tune.py ${vars.base_model} ${vars.output_model_suffix}'
 
   - name: fine_tune_status
     script:
-      - 'python ./project_scripts/fine_tune_status.py'
+      - 'python ../../util/fine_tune/fine_tune_status.py'
 
   - name: fine_tune_stats
     outputs:
       - 'output/fine_tune_stats.csv'
     script:
-      - './project_scripts/fine_tune_stats.sh ${env.openai_api_key} "file-IJ8nRijlbPpsVJY7eYLwxog9" output/fine_tune_stats.csv'
+      - '../../util/fine_tune/fine_tune_stats.sh ${env.openai_api_key} "file-IJ8nRijlbPpsVJY7eYLwxog9" output/fine_tune_stats.csv'

linker/fine_tune/project_scripts/create_citation_input_for_fine_tuning.py

@@ -6,7 +6,7 @@
 import srsly
 from util.general import load_mongo_docs
 from linker.fine_tune.project_scripts import constants
-from langchain.chat_models.openai import _convert_message_to_dict
+from langchain.chat_models.openai import convert_message_to_dict
 from langchain.schema import HumanMessage, SystemMessage, AIMessage
 
 
@@ -86,7 +86,7 @@ def _generate_one_completion_format(doc, is_labeled=False):
 
     @staticmethod
     def serialize_messages(messages):
-        return {"messages": [_convert_message_to_dict(message) for message in messages]}
+        return {"messages": [convert_message_to_dict(message) for message in messages]}
 
     @staticmethod
     def _create_system_prompt():

linker/fine_tune/project_scripts/fine_tune_stats.sh → util/fine_tune/fine_tune_stats.sh

@@ -3,14 +3,14 @@
 # Check if required inputs are provided
 if [ $# -ne 3 ]; then
   echo "Error: Missing required parameters"
-  echo "Usage: ./fine_tune.sh <OPENAI_API_KEY> <FINE_TUNE_JOB_ID> <OUTPUT_FILE>"
+  echo "Usage: ./fine_tune.sh <OPENAI_API_KEY> <RESULTS_FILE_ID> <OUTPUT_FILE>"
   exit 1
 fi
 
 # Read input variables
 OPENAI_API_KEY=$1
-FINE_TUNE_JOB_ID=$2
+RESULTS_FILE_ID=$2
 OUTPUT_FILE=$3
 
-curl https://api.openai.com/v1/files/$FINE_TUNE_JOB_ID/content \
+curl https://api.openai.com/v1/files/RESULTS_FILE_ID/content \
   -H "Authorization: Bearer $OPENAI_API_KEY" > "${OUTPUT_FILE}"