drd_p1/run_tests.sh at main · RnD4ASI/drd_p1 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
#!/bin/bash

# Run Tests for Data Attribute Deduplication System
# This script runs both standard and incremental deduplication tests

# Check if .env file exists (required for Azure OpenAI)
if [ ! -f ".env" ] && grep -q "provider: \"azure_openai\"" config.yaml; then
    echo "Error: .env file not found but Azure OpenAI is configured in config.yaml"
    echo "Please create a .env file with Azure OpenAI credentials (copy from env template)"
    exit 1
fi

# Create directories if they don't exist
mkdir -p result
mkdir -p tmp

echo "======================================================="
echo "Data Attribute Deduplication System - Test Runner"
echo "======================================================="
echo ""

# Set timestamp for this test run
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
echo "Test run started at: $(date)"
echo ""

# Run standard deduplication on batch1
echo "Running standard deduplication on batch1..."
python -m src.main --input data/batch1.csv --run_id "${TIMESTAMP}_test"

# Check if the run was successful
if [ $? -ne 0 ]; then
    echo "Error: Standard deduplication failed. Exiting."
    exit 1
fi

echo ""
echo "Standard deduplication completed successfully."
echo "Results saved to: result/${TIMESTAMP}_test/"
echo ""


# Print summary
echo "======================================================="
echo "Test Run Summary"
echo "======================================================="
echo "Timestamp: ${TIMESTAMP}"
echo "Standard deduplication: result/${TIMESTAMP}_test/"
echo ""
echo "Latest results are also available at:"
echo "- result/${TIMESTAMP}_test/deduplication_results.csv"
echo ""
echo "To view detailed results, check the run_metadata.json and deduplication.log files in each result directory."
echo "======================================================="


# Set timestamp for this test run
PRIOR_TIMESTAMP=${TIMESTAMP}
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
echo "Test run started at: $(date)"
echo ""

# Run incremental deduplication on batch2
echo "Running incremental deduplication on batch2..."
python -m src.main --input data/batch2.csv --previous_results result/${PRIOR_TIMESTAMP}_test/deduplication_results.csv --incremental --run_id "${TIMESTAMP}_test"

# Check if the run was successful
if [ $? -ne 0 ]; then
    echo "Error: Incremental deduplication failed. Exiting."
    exit 1
fi

echo ""
echo "Incremental deduplication completed successfully."
echo "Results saved to: result/${TIMESTAMP}_test/"
echo ""

# Print summary
echo "======================================================="
echo "Test Run Summary"
echo "======================================================="
echo "Timestamp: ${TIMESTAMP}"
echo "Incremental deduplication: result/${TIMESTAMP}_test/"
echo ""
echo "Latest results are also available at:"
echo "- result/${TIMESTAMP}_test/deduplication_results.csv"
echo ""
echo "To view detailed results, check the run_metadata.json and deduplication.log files in each result directory."
echo "======================================================="