Improvements on ms3 transform & convert; bug fixes (#127)

Author: johentsch

src/ms3/bs4_parser.py

@@ -2683,9 +2683,9 @@ def parse_measures(self):
                                 event["mc_onset"] = current_position
                                 chord_info["mc_onset"] = current_position
                             if tremolo_type:
-                                chord_info[
-                                    "tremolo"
-                                ] = f"{tremolo_duration_string}_{tremolo_type}_{tremolo_component}"
+                                chord_info["tremolo"] = (
+                                    f"{tremolo_duration_string}_{tremolo_type}_{tremolo_component}"
+                                )
                                 if tremolo_component in (0, 2):
                                     # delete 'tremolo_type' which signals that the <Chord> is part of a tremolo
                                     tremolo_type = None
@@ -2793,7 +2793,9 @@ def safe_update_event(key, value):
                                 # block
                                 parent_name = text_tag.parent.name
                                 text_including_html = text_tag2str(text_tag)
-                                text_excluding_html = text_tag2str_recursive(text_tag)
+                                text_excluding_html = text_tag2str_recursive(
+                                    text_tag, join_char=" "
+                                )
                                 if parent_name == "Fingering":
                                     # fingerings occur within <Note> tags, if they are to be extracted, they should go
                                     # into the notes table
@@ -3304,14 +3306,14 @@ def replace_first_harmony(self, first_harmony_values: Dict[str, str]):
         )
 
     @overload
-    def get_onset_zero_harmony(self, return_layer: Literal[False]) -> Optional[bs4.Tag]:
-        ...
+    def get_onset_zero_harmony(
+        self, return_layer: Literal[False]
+    ) -> Optional[bs4.Tag]: ...
 
     @overload
     def get_onset_zero_harmony(
         self, return_layer: Literal[True]
-    ) -> Tuple[Optional[bs4.Tag], int, int]:
-        ...
+    ) -> Tuple[Optional[bs4.Tag], int, int]: ...
 
     def get_onset_zero_harmony(self, return_layer: bool = False) -> Optional[bs4.Tag]:
         """Iterate through all tags at mc_onset 0 for all notational (staff, voice) layers and return the first
@@ -3618,10 +3620,10 @@ def __repr__(self):
     ),
     index_col=0,
 )
-INSTRUMENT_DEFAULTS[
-    ["controllers", "ChannelName", "ChannelValue"]
-] = INSTRUMENT_DEFAULTS[["controllers", "ChannelName", "ChannelValue"]].apply(
-    lambda k: list(map(lambda j: eval(j) if j is not None else None, k))
+INSTRUMENT_DEFAULTS[["controllers", "ChannelName", "ChannelValue"]] = (
+    INSTRUMENT_DEFAULTS[["controllers", "ChannelName", "ChannelValue"]].apply(
+        lambda k: list(map(lambda j: eval(j) if j is not None else None, k))
+    )
 )
 for int_column in ["keysig", "useDrumset"]:
     INSTRUMENT_DEFAULTS[int_column] = INSTRUMENT_DEFAULTS[int_column].astype("Int64")
@@ -3875,9 +3877,9 @@ def modify_list_tags(self, changed_part, found, value):
         :param value: new values to set
         :return: corrected list of parts of the same length as value list
         """
-        l_found, l_value = 1 if found is None else len(
-            found
-        ), 1 if value is None else len(value)
+        l_found, l_value = 1 if found is None else len(found), (
+            1 if value is None else len(value)
+        )
         if l_found < l_value:
             for i in range(l_value - l_found):
                 new_tag = self.soup.new_tag("Channel")
@@ -4796,15 +4798,13 @@ def tag2text(tag: bs4.Tag) -> Tuple[str, str]:
 @overload
 def find_tag_get_string(
     parent_tag: bs4.Tag, tag_to_find: str, fallback: Literal[None]
-) -> Tuple[Optional[bs4.Tag], Optional[str]]:
-    ...
+) -> Tuple[Optional[bs4.Tag], Optional[str]]: ...
 
 
 @overload
 def find_tag_get_string(
     parent_tag: bs4.Tag, tag_to_find: str, fallback: Hashable
-) -> Tuple[Optional[bs4.Tag], Optional[Hashable]]:
-    ...
+) -> Tuple[Optional[bs4.Tag], Optional[Hashable]]: ...
 
 
 def find_tag_get_string(
@@ -4885,13 +4885,13 @@ def process_thoroughbass(
 @overload
 def get_row_at_quarterbeat(
     df: pd.DataFrame, quarterbeat: Literal[None]
-) -> pd.DataFrame:
-    ...
+) -> pd.DataFrame: ...
 
 
 @overload
-def get_row_at_quarterbeat(df: pd.DataFrame, quarterbeat: float) -> Optional[pd.Series]:
-    ...
+def get_row_at_quarterbeat(
+    df: pd.DataFrame, quarterbeat: float
+) -> Optional[pd.Series]: ...
 
 
 def get_row_at_quarterbeat(

src/ms3/cli.py

@@ -16,6 +16,7 @@
 from ms3 import (
     Parse,
     compute_path_from_file,
+    convert_from_metadata_tsv,
     get_git_repo,
     get_git_version_info,
     make_coloring_reports_and_warnings,
@@ -164,7 +165,6 @@ def convert_cmd(args):
     #    '\nTARGET_DIR: ' + target
     update_logger = get_logger("ms3.convert", level=args.level)
     for argument_name, argument, default in (
-        ("-a/--all", args.all, False),
         ("-e/--exclude", args.exclude, None),
         ("-f/--folders", args.folders, None),
         ("--reviewed", args.reviewed, False),
@@ -175,22 +175,38 @@ def convert_cmd(args):
             update_logger.info(
                 f"Argument '{argument_name}' is currently being ignored."
             )
-    out_dir = os.getcwd() if args.out is None else resolve_dir(args.out)
+    out_dir = "." if args.out is None else args.out
     ms = "auto" if args.musescore is None else args.musescore
-    convert_folder(
-        directory=args.dir,
-        file_paths=args.files,
-        target_dir=out_dir,
-        extensions=args.extensions,
-        target_extension=args.format,
-        regex=args.include,
-        suffix=args.suffix,
-        recursive=not args.nonrecursive,
-        ms=ms,
-        overwrite=args.safe,
-        parallel=not args.iterative,
-        logger=update_logger,
-    )
+    if args.all:
+        convert_folder(
+            directory=args.dir,
+            file_paths=args.files,
+            target_dir=out_dir,
+            extensions=args.extensions,
+            target_extension=args.format,
+            regex=args.include,
+            suffix=args.suffix,
+            recursive=not args.nonrecursive,
+            ms=ms,
+            overwrite=args.safe,
+            parallel=not args.iterative,
+            logger=update_logger,
+        )
+    else:
+        convert_from_metadata_tsv(
+            directory=args.dir,
+            file_paths=args.files,
+            target_dir=out_dir,
+            extensions=args.extensions,
+            target_extension=args.format,
+            regex=args.include,
+            suffix=args.suffix,
+            recursive=not args.nonrecursive,
+            ms=ms,
+            overwrite=args.safe,
+            parallel=not args.iterative,
+            logger=update_logger,
+        )
 
 
 def empty(args, parse_obj: Optional[Parse] = None):
@@ -290,12 +306,15 @@ def transform_cmd(args):
     if repo is None:
         version_info = None
     elif repo.is_dirty():
-        print(
-            "The repository is dirty. Please commit or stash your changes before running ms3 transform. This is "
-            "important because the version information in the JSON descriptor(s) needs to be consistent with the "
-            "repository state."
-        )
-        return
+        if args.dirty:
+            version_info = None
+        else:
+            print(
+                "The repository is dirty. Please commit or stash your changes before running ms3 transform. This is "
+                "important because the version information in the JSON descriptor(s) needs to be consistent with the "
+                "repository state. To ignore this warning, add the --dirty flag to the command."
+            )
+            return
     else:
         version_info = get_git_version_info(repo=repo)
     parse_obj = make_parse_obj(args, parse_tsv=True, facets=params)
@@ -339,18 +358,22 @@ def update_cmd(args, parse_obj: Optional[Parse] = None):
         print("Nothing to do.")
 
 
-def check_and_create(d):
+def check_and_create(d, resolve=True):
     """Turn input into an existing, absolute directory path."""
     if not os.path.isdir(d):
-        d = resolve_dir(os.path.join(os.getcwd(), d))
-        if not os.path.isdir(d):
-            if input(d + " does not exist. Create? (y|n)") == "y":
-                os.mkdir(d)
+        d_resolved = resolve_dir(os.path.join(os.getcwd(), d))
+        if not os.path.isdir(d_resolved):
+            if input(d_resolved + " does not exist. Create? (y|n)") == "y":
+                os.mkdir(d_resolved)
             else:
                 raise argparse.ArgumentTypeError(
                     d + " needs to be an existing directory"
                 )
-    return resolve_dir(d)
+    return d_resolved if resolve else d
+
+
+def check_and_create_unresolved(d):
+    return check_and_create(d, resolve=False)
 
 
 def check_dir(d):
@@ -381,13 +404,11 @@ def precommit_cmd(
 
 
 @overload
-def review_cmd(args, parse_obj, wrapped_by_precommit: Literal[False]) -> None:
-    ...
+def review_cmd(args, parse_obj, wrapped_by_precommit: Literal[False]) -> None: ...
 
 
 @overload
-def review_cmd(args, parse_obj, wrapped_by_precommit: Literal[True]) -> bool:
-    ...
+def review_cmd(args, parse_obj, wrapped_by_precommit: Literal[True]) -> bool: ...
 
 
 def review_cmd(
@@ -627,8 +648,9 @@ def get_arg_parser():
         "-o",
         "--out",
         metavar="OUT_DIR",
-        type=check_and_create,
-        help="Output directory.",
+        type=check_and_create_unresolved,
+        help="Output directory. For conversion, an absolute path will result in a copy of the original sub-folder "
+        "structure, whereas a relative path will contain all converted files next to each other.",
     )
     parse_args.add_argument(
         "-n",
@@ -1006,7 +1028,7 @@ def get_arg_parser():
         "--format",
         default="mscx",
         help="Output format of converted files. Defaults to mscx. Other options are "
-        "{png, svg, pdf, mscz, wav, mp3, flac, ogg, xml, mxl, mid}",
+        "{png, svg, pdf, mscz, wav, mp3, flac, ogg, musicxml, mxl, mid}",
     )
     convert_parser.add_argument(
         "--extensions",
@@ -1196,6 +1218,11 @@ def get_arg_parser():
         action="store_true",
         help="Store the transformed files as uncompressed TSVs rather than writing them into a ZIP file.",
     )
+    transform_parser.add_argument(
+        "--dirty",
+        action="store_true",
+        help="Allows to override the 'This repository is dirty' blocker.",
+    )
     transform_parser.set_defaults(func=transform_cmd)
 
     update_parser = subparsers.add_parser(

src/ms3/expand_dcml.py

@@ -1,6 +1,7 @@
-""" This is the same code as in the corpora repo as copied on September 24, 2020
+"""This is the same code as in the corpora repo as copied on September 24, 2020
 and then adapted.
 """
+
 import re
 import sys
 from collections import defaultdict
@@ -709,7 +710,7 @@ def make_comparison():
             if n_beginnings == n_endings_cleaned:
                 mismatch_maybe_due_to_voltas = True
                 logger.info(
-                    "One or several pedal points have there endings in a first/second ending scenario. "
+                    "One or several pedal points have their endings in a first/second ending scenario. "
                     "So far I can only correctly propagate the pedal note into first endings, not the others."
                 )
     if mismatch_maybe_due_to_voltas is False:
@@ -746,12 +747,14 @@ def make_comparison():
             # if the localkey changes during the pedal point, the reference changes and the Roman numeral indicating
             # the pedal note needs to be adapted
             key2pedal = {
-                key: ped
-                if key == first_localkey
-                else abs2rel_key(
-                    rel2abs_key(ped, first_localkey, global_minor, logger=logger),
-                    key,
-                    global_minor,
+                key: (
+                    ped
+                    if key == first_localkey
+                    else abs2rel_key(
+                        rel2abs_key(ped, first_localkey, global_minor, logger=logger),
+                        key,
+                        global_minor,
+                    )
                 )
                 for key in localkeys.unique()
             }

src/ms3/operations.py

@@ -1,6 +1,7 @@
 """This module contains the functions called by the ms3 commandline interface, which is why they may use
 print() instead of log messages from time to time.
 """
+
 import os
 from typing import Dict, Iterator, List, Literal, Optional, Tuple, Union
 
@@ -25,6 +26,7 @@
     store_dataframe_resource,
     store_dataframes_package,
     tpc2scale_degree,
+    update_relative_paths_with_corpus_dirs,
     write_tsv,
 )
 from ms3.utils.constants import LATEST_MUSESCORE_VERSION
@@ -457,6 +459,7 @@ def _transform(
         # get concatenated dataframe:
         if facet == "metadata":
             df = ms3_object.metadata()
+            update_relative_paths_with_corpus_dirs(df)
         else:
             if obj_is_corpus:
                 df = ms3_object.get_facet(

src/ms3/piece.py

@@ -43,6 +43,7 @@
     get_git_version_info,
     get_musescore,
     infer_tsv_type,
+    literal_type2tuple,
     load_tsv,
     make_file_path,
     metadata2series,
@@ -212,17 +213,15 @@ def score_metadata(
         view_name: Optional[str],
         choose: Literal["auto", "ask"],
         as_dict: Literal[False],
-    ) -> pd.Series:
-        ...
+    ) -> pd.Series: ...
 
     @overload
     def score_metadata(
         self,
         view_name: Optional[str],
         choose: Literal["auto", "ask"],
         as_dict: Literal[True],
-    ) -> dict:
-        ...
+    ) -> dict: ...
 
     def score_metadata(
         self,
@@ -1331,8 +1330,8 @@ def get_parsed_tsv(
     ) -> FileDataframeTupleMaybe:
         facets = argument_and_literal_type2list(facet, TSVtype, logger=self.logger)
         assert (
-            len(facet) == 1
-        ), f"Pass exactly one valid TSV type {TSVtype.__args__} or use _.get_parsed_tsvs()"
+            len(facets) == 1
+        ), f"Pass exactly one valid TSV type {literal_type2tuple(TSVtype)} or use _.get_parsed_tsvs()\nGot: {facets}"
         facet = facets[0]
         return self.get_parsed(facet, view_name=view_name, choose=choose)