Skip to content

Commit 48eac94

Browse files
vish-csvish-cs
committed
Update differ to ouptut MCF files
1 parent 6c55a97 commit 48eac94

17 files changed

Lines changed: 535 additions & 347 deletions

‎tools/import_differ/differ_utils.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,28 @@ def write_json_data(data, dest: str, file: str, tmp_dir: str):
8484
upload_output_data(path, dest)
8585

8686

87+
def write_mcf_nodes(nodes: list, dest: str, file: str, tmp_dir: str):
88+
""" Writes mcf nodes to a file with the given path."""
89+
if dest.startswith('gs://'):
90+
path = os.path.join(tmp_dir, file)
91+
else:
92+
path = os.path.join(dest, file)
93+
with open(path, mode='w', encoding='utf-8') as out_file:
94+
for node in nodes:
95+
if 'Node' in node:
96+
out_file.write(f'Node: {node["Node"]}\n')
97+
elif 'dcid' in node:
98+
out_file.write(f'dcid: {node["dcid"]}\n')
99+
100+
for key, value in node.items():
101+
if key in ['Node', 'dcid']:
102+
continue
103+
out_file.write(f'{key}: {value}\n')
104+
out_file.write('\n')
105+
if dest.startswith('gs://'):
106+
upload_output_data(path, dest)
107+
108+
87109
def upload_output_data(src: str, dest: str):
88110
client = storage.Client()
89111
bucket_name = dest.split('/')[2]

‎tools/import_differ/import_differ.py‎

Lines changed: 170 additions & 126 deletions
Large diffs are not rendered by default.

‎tools/import_differ/import_differ_test.py‎

Lines changed: 63 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
import json
1516
import os
1617
import pandas as pd
1718
import unittest
@@ -32,27 +33,68 @@ class TestImportDiffer(unittest.TestCase):
3233
def test_diff_analysis(self):
3334
current_data = os.path.join(module_dir, 'test', 'current', '*.mcf')
3435
previous_data = os.path.join(module_dir, 'test', 'previous', '*.mcf')
35-
output_location = os.path.join(module_dir)
36-
self.differ = import_differ.ImportDiffer(current_data, previous_data,
37-
output_location)
38-
current_mcf = differ_utils.load_data(self.differ.current_data, '.')
39-
previous_mcf = differ_utils.load_data(self.differ.previous_data, '.')
40-
current_obs_df, current_schema_df = self.differ.split_data(current_mcf)
41-
previous_obs_df, previous_schema_df = self.differ.split_data(
42-
previous_mcf)
43-
obs_diff = self.differ.generate_diff(previous_obs_df, current_obs_df)
44-
summary, _ = self.differ.observation_diff_analysis(obs_diff)
45-
expected_summary = pd.read_csv(
46-
os.path.join(module_dir, 'test', 'results', 'obs_diff_summary.csv'))
47-
assert_frame_equal(summary, expected_summary)
48-
49-
schema_diff = self.differ.generate_diff(previous_schema_df,
50-
current_schema_df)
51-
summary = self.differ.schema_diff_analysis(schema_diff)
52-
expected_summary = pd.read_csv(
53-
os.path.join(module_dir, 'test', 'results',
54-
'schema_diff_summary.csv'))
55-
assert_frame_equal(summary, expected_summary)
36+
output_location = os.path.join(module_dir, 'test', 'output')
37+
if os.path.exists(output_location):
38+
import shutil
39+
shutil.rmtree(output_location)
40+
os.makedirs(output_location)
41+
42+
differ = import_differ.ImportDiffer(current_data=current_data,
43+
previous_data=previous_data,
44+
output_location=output_location,
45+
runner_mode='native')
46+
differ.run_differ()
47+
48+
# Check for expected files
49+
expected_files = ['import_diff.mcf', 'differ_summary.json']
50+
51+
for f in expected_files:
52+
file_path = os.path.join(output_location, f)
53+
self.assertTrue(os.path.exists(file_path),
54+
f"File {f} was not generated")
55+
56+
# Verify content of the combined MCF file
57+
with open(os.path.join(output_location, 'import_diff.mcf'), 'r') as f:
58+
content = f.read().strip()
59+
60+
with open(
61+
os.path.join(module_dir, 'test', 'results', 'import_diff.mcf'),
62+
'r') as f:
63+
expected_content = f.read().strip()
64+
65+
# Split into individual nodes, strip whitespace, and sort to avoid ordering issues
66+
actual_nodes = sorted(
67+
[node.strip() for node in content.split('\n\n') if node.strip()])
68+
expected_nodes = sorted([
69+
node.strip()
70+
for node in expected_content.split('\n\n')
71+
if node.strip()
72+
])
73+
74+
self.assertListEqual(actual_nodes, expected_nodes)
75+
76+
# Verify content of the summary file
77+
with open(os.path.join(output_location, 'differ_summary.json'),
78+
'r') as f:
79+
summary_content = json.load(f)
80+
81+
with open(
82+
os.path.join(module_dir, 'test', 'results',
83+
'differ_summary.json'), 'r') as f:
84+
expected_summary_content = json.load(f)
85+
86+
# The version paths might differ based on where the test is run,
87+
# so we normalize or remove them for the comparison
88+
summary_content.pop('current_version', None)
89+
summary_content.pop('previous_version', None)
90+
expected_summary_content.pop('current_version', None)
91+
expected_summary_content.pop('previous_version', None)
92+
93+
self.assertDictEqual(summary_content, expected_summary_content)
94+
95+
if os.path.exists(output_location):
96+
import shutil
97+
shutil.rmtree(output_location)
5698

5799

58100
if __name__ == '__main__':

‎tools/import_differ/test/current/schema.mcf‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,71 @@
1-
dcid: "InterestRate_TreasuryBill_6Month"
1+
Node: dcid:InterestRate_TreasuryBill_6Month
22
name: "InterestRate_TreasuryBill_6Month"
33
typeOf: dcs:StatisticalVariable
44
measuredProperty: dcs:interestRate
55
populationType: dcs:TreasuryBill
66
maturity: [6 Month]
77
statType: dcs:measuredValue
88

9-
dcid: "InterestRate_TreasuryBill_1Year"
9+
Node: dcid:InterestRate_TreasuryBill_1Year
1010
name: "InterestRate_TreasuryBill_1Year"
1111
typeOf: dcs:StatisticalVariable
1212
measuredProperty: dcs:interestRate
1313
populationType: dcs:TreasuryBill
1414
maturity: [1 Year]
1515
statType: dcs:measuredValue
1616

17-
dcid: "InterestRate_TreasuryNote_2Year"
17+
Node: dcid:InterestRate_TreasuryNote_2Year
1818
name: "InterestRate_TreasuryNote_2Year"
1919
typeOf: dcs:StatisticalVariable
2020
measuredProperty: dcs:interestRate
2121
populationType: dcs:TreasuryNote
2222
maturity: [2 Year]
2323
statType: dcs:measuredValue
2424

25-
dcid: "InterestRate_TreasuryNote_3Year"
25+
Node: dcid:InterestRate_TreasuryNote_3Year
2626
name: "InterestRate_TreasuryNote_3Year"
2727
typeOf: dcs:StatisticalVariable
2828
measuredProperty: dcs:interestRate
2929
populationType: dcs:TreasuryNote
3030
maturity: [3 Year]
3131
statType: dcs:measuredValue
3232

33-
dcid: "InterestRate_TreasuryNote_5Year"
33+
Node: dcid:InterestRate_TreasuryNote_5Year
3434
name: "InterestRate_TreasuryNote_5Year"
3535
typeOf: dcs:StatisticalVariable
3636
measuredProperty: dcs:interestRate
3737
populationType: dcs:TreasuryNote
3838
maturity: [5 Year]
3939
statType: dcs:measuredValue
4040

41-
dcid: "InterestRate_TreasuryNote_7Year"
41+
Node: dcid:InterestRate_TreasuryNote_7Year
4242
name: "InterestRate_TreasuryNote_7Year"
4343
typeOf: dcs:StatisticalVariable
4444
measuredProperty: dcs:interestRate
4545
populationType: dcs:TreasuryNote
4646
maturity: [7 Year]
4747
statType: dcs:measuredValue
4848

49-
dcid: "InterestRate_TreasuryNote_10Year"
49+
Node: dcid:InterestRate_TreasuryNote_10Year
5050
name: "InterestRate_TreasuryNote_10Year"
5151
typeOf: dcs:StatisticalVariable
5252
measuredProperty: dcs:interestRate
5353
populationType: dcs:TreasuryNote
5454
maturity: [10 Year]
5555
statType: dcs:measuredValue
5656

57-
dcid: "InterestRate_TreasuryBond_20Year"
57+
Node: dcid:InterestRate_TreasuryBond_20Year
5858
name: "InterestRate_TreasuryBond_20Year"
5959
typeOf: dcs:StatisticalVariable
6060
measuredProperty: dcs:interestRate
6161
populationType: dcs:TreasuryBond
6262
maturity: [20 Year]
6363
statType: dcs:measuredValue
6464

65-
dcid: "InterestRate_TreasuryBond_30Year"
65+
Node: dcid:InterestRate_TreasuryBond_30Year
6666
name: "InterestRate_TreasuryBond_30Year"
6767
typeOf: dcs:StatisticalVariable
6868
measuredProperty: dcs:interestRate
6969
populationType: dcs:TreasuryBond
7070
maturity: [30 Year]
7171
statType: dcs:measuredValue
72-

‎tools/import_differ/test/previous/schema.mcf‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,63 @@
1-
dcid: "InterestRate_TreasuryBill_1Month"
1+
Node: dcid:InterestRate_TreasuryBill_1Month
22
name: "InterestRate_TreasuryBill_1Month"
33
typeOf: dcs:StatisticalVariable
44
measuredProperty: dcs:interestRate
55
populationType: dcs:TreasuryBill
66
maturity: [1 Month]
77
statType: dcs:measuredValue
88

9-
dcid: "InterestRate_TreasuryBill_3Month"
9+
Node: dcid:InterestRate_TreasuryBill_3Month
1010
name: "InterestRate_TreasuryBill_3Month"
1111
typeOf: dcs:StatisticalVariable
1212
measuredProperty: dcs:interestRate
1313
populationType: dcs:TreasuryBill
1414
maturity: [3 Month]
1515
statType: dcs:measuredValue
1616

17-
dcid: "InterestRate_TreasuryBill_6Month"
17+
Node: dcid:InterestRate_TreasuryBill_6Month
1818
name: "InterestRate_TreasuryBill_6Month"
1919
typeOf: dcs:StatisticalVariable
2020
measuredProperty: dcs:interestRate
2121
populationType: dcs:TreasuryBill
2222
maturity: [6 Month]
2323
statType: dcs:measuredValue
2424

25-
dcid: "InterestRate_TreasuryBill_1Year"
25+
Node: dcid:InterestRate_TreasuryBill_1Year
2626
name: "InterestRate_TreasuryBill_01Year"
2727
typeOf: dcs:StatisticalVariable
2828
measuredProperty: dcs:interestRate
2929
populationType: dcs:TreasuryBill
3030
maturity: [1 Year]
3131
statType: dcs:measuredValue
3232

33-
dcid: "InterestRate_TreasuryNote_2Year"
33+
Node: dcid:InterestRate_TreasuryNote_2Year
3434
name: "InterestRate_TreasuryNote_02Year"
3535
typeOf: dcs:StatisticalVariable
3636
measuredProperty: dcs:interestRate
3737
populationType: dcs:TreasuryNote
3838
maturity: [2 Year]
3939
statType: dcs:measuredValue
4040

41-
dcid: "InterestRate_TreasuryNote_10Year"
41+
Node: dcid:InterestRate_TreasuryNote_10Year
4242
name: "InterestRate_TreasuryNote_10Year"
4343
typeOf: dcs:StatisticalVariable
4444
measuredProperty: dcs:interestRate
4545
populationType: dcs:TreasuryNote
4646
maturity: [10 Year]
4747
statType: dcs:measuredValue
4848

49-
dcid: "InterestRate_TreasuryBond_20Year"
49+
Node: dcid:InterestRate_TreasuryBond_20Year
5050
name: "InterestRate_TreasuryBond_20Year"
5151
typeOf: dcs:StatisticalVariable
5252
measuredProperty: dcs:interestRate
5353
populationType: dcs:TreasuryBond
5454
maturity: [20 Year]
5555
statType: dcs:measuredValue
5656

57-
dcid: "InterestRate_TreasuryBond_30Year"
57+
Node: dcid:InterestRate_TreasuryBond_30Year
5858
name: "InterestRate_TreasuryBond_30Year"
5959
typeOf: dcs:StatisticalVariable
6060
measuredProperty: dcs:interestRate
6161
populationType: dcs:TreasuryBond
6262
maturity: [30 Year]
63-
statType: dcs:measuredValue
63+
statType: dcs:measuredValue

‎tools/import_differ/test/results/differ_summary.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"current_version": "tools/import_differ/test/current/*.mcf",
3+
"previous_version": "tools/import_differ/test/previous/*.mcf",
4+
"current_obs_count": 20,
5+
"previous_obs_count": 21,
6+
"current_schema_count": 9,
7+
"previous_schema_count": 8,
8+
"added_obs_count": 1,
9+
"deleted_obs_count": 2,
10+
"modified_obs_count": 1,
11+
"added_schema_count": 3,
12+
"deleted_schema_count": 2,
13+
"modified_schema_count": 2,
14+
"obs_diff_count": 4,
15+
"schema_diff_count": 7
16+
}

0 commit comments

Comments
 (0)