test model dump and fix, add tests to pr workflow (#101)

* test model dump and fix, add tests to pr workflow

* use composite action for setup

Author: mingness

.github/actions/setup/action.yml

@@ -0,0 +1,13 @@
+name: "Setup Python"
+description: "Setup Python and dependencies"
+runs:
+  using: "composite"
+  steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '>=3.11'
+
+      - name: Install dependencies
+        shell: bash
+        run: pip install -r requirements.txt

.github/workflows/issue_to_pr.yml

@@ -31,12 +31,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.x
-
-      - name: Install dependencies
-        run: pip install -r requirements.txt
+        uses: ./.github/actions/setup
 
       - name: Read and validate properties txt file
         id: parseProps

.github/workflows/pull_request_test_run_update_flow.yml

@@ -0,0 +1,30 @@
+name: Pull Request Test Update Workflow
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  test_run:
+    name: Test that update contributions workflow runs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: ./.github/actions/setup
+
+      - name: run unit tests
+        run: pytest
+
+      - name: fetch updates on contributions
+        run: python -u scripts/fetch_updates.py
+
+      - name: write contribs.txt file
+        run: python -u scripts/to_contribs_txt.py
+
+      - name: write source json files
+        run: python -u scripts/to_sources_jsons.py

.github/workflows/update_contributions.yml

@@ -15,12 +15,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '>=3.11'
-
-      - name: Install dependencies
-        run: pip install -r requirements.txt
+        uses: ./.github/actions/setup
 
       - name: fetch updates and edit database
         run: |

scripts/add_new_contribution_to_yaml.py

@@ -8,22 +8,31 @@
 
 from ruamel.yaml import YAML
 
+
+def split_categories(categories):
+    categories = sorted(categories.replace('"', '').split(','))
+    categories = [category.strip() for category in categories]
+    return categories
+
+
+def postprocess_properties(properties_dict):
+    if 'categories' in properties_dict and properties_dict['categories']:
+        properties_dict['categories'] = split_categories(properties_dict['categories'])
+    else:
+        properties_dict['categories'] = None
+
+    # add download
+    if 'download' not in properties_dict:
+        properties_dict['download'] = properties_dict['source'][:properties_dict['source'].rfind('.')] + '.zip'
+
+
 if __name__ == "__main__":
     if len(argv) < 2:
         print("script takes json string as argument.\nStopping...")
         raise ValueError
 
     props = json.loads(argv[1])
-    # process category list
-    if 'categories' in props and props['categories']:
-        props['categories'] = sorted(props['categories'].replace('"', '').split(','))
-        props['categories'] = [category.strip() for category in props['categories']]
-    else:
-        props['categories'] = None
-
-    # add download
-    if 'download' not in props:
-        props['download'] = props['source'][:props['source'].rfind('.')] + '.zip'
+    postprocess_properties(props)
 
     # open database
     database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml'

scripts/fetch_updates.py

@@ -78,6 +78,18 @@ def process_contribution(item):
   return index, contribution
 
 
+def process_all(contributions_list):
+  total = len(contributions_list)
+  completed = 0
+  print(f"Starting processing of {total} contributions...")
+
+  with Pool(processes=256) as pool:
+    for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)):
+      contributions_list[index] = contribution
+      completed += 1
+      print(f"Progress: {completed}/{total} ({(completed / total * 100):.1f}%)")
+
+
 if __name__ == "__main__":
   parser = argparse.ArgumentParser()
   parser.add_argument('--index')
@@ -97,16 +109,7 @@ def process_contribution(item):
   contributions_list = data['contributions']
 
   if index == 'all':
-    total = len(contributions_list)
-    completed = 0
-    print(f"Starting processing of {total} contributions...")
-    
-    with Pool(processes=256) as pool:
-        for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)):
-            contributions_list[index] = contribution
-            completed += 1
-            print(f"Progress: {completed}/{total} ({(completed/total*100):.1f}%)")
-    
+    process_all(contributions_list)
     print("All processing complete")
   else:
     # update only contribution with id==index

scripts/parse_and_validate_properties_txt.py

@@ -75,19 +75,19 @@ def validate_existing(properties_dict):
     # validation on existing contribution is weaker
     properties = PropertiesExisting.model_validate(properties_dict)
 
-    return properties.model_dump(exclude_unset=True)
+    return properties.model_dump()
 
 def validate_new(properties_dict):
     # new contribution has stronger validation
     properties = PropertiesBase.model_validate(properties_dict)
 
-    return properties.model_dump(exclude_unset=True)
+    return properties.model_dump()
 
 def validate_new_library(properties_dict):
     # new contribution has stronger validation
     properties = LibraryPropertiesNew.model_validate(properties_dict)
 
-    return properties.model_dump(exclude_unset=True)
+    return properties.model_dump()
 
 def set_output(output_object):
     with open(os.environ['GITHUB_OUTPUT'],'a') as f:

scripts/to_contribs_txt.py

@@ -6,6 +6,7 @@
 import pathlib
 import shutil
 from collections import defaultdict
+from typing import List
 
 from utils import get_valid_contributions
 
@@ -45,6 +46,40 @@ def read_contribs_text(filepath):
   return contribs_list
 
 
+def preprocess_contributions() -> List:
+  all_contributions = get_valid_contributions()
+
+  # sort contributions list by type
+  def sort_key(d):
+    return type_list.index(d['type'])
+  all_contributions = sorted(all_contributions, key=sort_key)
+
+  return all_contributions
+
+
+def write_contribs(all_contributions, fh):
+  for contribution in all_contributions:
+    fh.write(contribution['type'] + '\n')
+    for field in contribs_fields_list:
+      if field in contribution:
+        if field == 'id':
+          fh.write(f'{field}={contribution[field]:03}\n')
+        elif field == 'categories':
+          if contribution['type'] == 'library':
+            fh.write(f'{field}={",".join(contribution[field]) if contribution[field] else ""}\n')
+          else:
+            # categories are only relevant for libraries, except for examples with "Books" as category
+            if contribution[field] and 'Books' in contribution[field]:
+              fh.write(f'{field}={",".join(contribution[field]) if contribution[field] else ""}\n')
+            else:
+              fh.write(f'{field}=\n')
+        elif field == 'compatibleModesList':
+          fh.write(f'modes={contribution[field]}\n')
+        else:
+          fh.write(f'{field}={"" if contribution[field] is None else contribution[field]}\n')
+    fh.write('\n')
+
+
 if __name__ == "__main__":
   pde_folder = pathlib.Path(__file__).parent.parent / 'pde/'
   # remove sources folder if it already exists
@@ -54,34 +89,10 @@ def read_contribs_text(filepath):
 
   contribs_text_file = pde_folder / 'contribs.txt'
 
-  contributions_list = get_valid_contributions()
-
-  # sort contributions list by type
-  def sort_key(d):
-    return type_list.index(d['type'])
-  contributions_list = sorted(contributions_list, key=sort_key)
+  contributions_list = preprocess_contributions()
 
   # write contribs.txt file
   with open(contribs_text_file, 'w+') as f:
-    for contribution in contributions_list:
-      f.write(contribution['type']+'\n')
-      for field in contribs_fields_list:
-        if field in contribution:
-          if field == 'id':
-            f.write(f'{field}={contribution[field]:03}\n')
-          elif field == 'categories':
-            if contribution['type'] == 'library':
-              f.write(f'{field}={",".join(contribution[field]) if contribution[field] else ""}\n')
-            else:
-              # categories are only relevant for libraries, except for examples with "Books" as category
-              if contribution[field] and 'Books' in contribution[field]:
-                f.write(f'{field}={",".join(contribution[field]) if contribution[field] else ""}\n')
-              else:
-                f.write(f'{field}=\n')
-          elif field == 'compatibleModesList':
-            f.write(f'modes={contribution[field]}\n')
-          else:
-            f.write(f'{field}={"" if contribution[field] is None else contribution[field]}\n')
-      f.write('\n')
+    write_contribs(contributions_list, f)
 
 

scripts/to_sources_jsons.py

@@ -63,6 +63,17 @@ def to_sources_dict(contribution_dict):
   return sources_dict
 
 
+def write_json_for_each_contribution_in_list(all_contributions, folder_path):
+    for contribution in all_contributions:
+        if 'name' in contribution:
+            # output zero padded string for id
+            contribution['id'] = f"{contribution['id']:03}"
+            filename = contribution['name'].replace(':', '').replace('/', '').replace(' ', '_') + '.json'
+            this_filepath = folder_path / filename
+            with open(this_filepath, 'w') as f:
+                json.dump(to_sources_dict(contribution), f, indent=2)
+
+
 if __name__ == "__main__":
   sources_folder = pathlib.Path(__file__).parent.parent / 'sources/'
 
@@ -74,11 +85,4 @@ def to_sources_dict(contribution_dict):
   sources_folder.mkdir(parents=True, exist_ok=True)
 
   # create a json file in the sources folder for each contribution
-  for contribution in contributions_list:
-    if 'name' in contribution:
-      # output zero padded string for id
-      contribution['id'] = f"{contribution['id']:03}"
-      filename = contribution['name'].replace(':','').replace('/','').replace(' ','_') + '.json'
-      this_filepath = sources_folder / filename
-      with open(this_filepath, 'w') as f:
-        json.dump(to_sources_dict(contribution),f,indent=2)
+  write_json_for_each_contribution_in_list(contributions_list, sources_folder)