From e755a6c728832a792d3a51d6c7823569172bda14 Mon Sep 17 00:00:00 2001 From: sven-oly Date: Tue, 4 Mar 2025 14:34:19 -0800 Subject: [PATCH 01/13] Starting to add serial flag for schema testing --- schema/check_generated_data.py | 5 ++++- schema/check_schemas.py | 37 ++++++++++++++++++++++++---------- schema/check_test_output.py | 4 ++++ testdriver/ddtargs.py | 15 ++++++++++++++ 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/schema/check_generated_data.py b/schema/check_generated_data.py index df433de8..3daed44d 100644 --- a/schema/check_generated_data.py +++ b/schema/check_generated_data.py @@ -12,6 +12,10 @@ import os.path import sys +# To get commanlin arguments +sys.path.append('../testdriver') +from ddtargs import schemaArgs + import schema_validator from schema_files import ALL_TEST_TYPES @@ -101,6 +105,5 @@ def main(args): logging.info("All %d generated test data files match with schema", schema_count) - if __name__ == "__main__": main(sys.argv) diff --git a/schema/check_schemas.py b/schema/check_schemas.py index c16865cc..223d00e8 100644 --- a/schema/check_schemas.py +++ b/schema/check_schemas.py @@ -15,6 +15,10 @@ import schema_validator from schema_files import ALL_TEST_TYPES +# To get commanlin arguments +sys.path.append('../testdriver') +from ddtargs import schemaArgs + class ValidateSchema: def __init__(self, schema_base='.'): @@ -61,17 +65,26 @@ def save_schema_validation_summary(self, validation_status): def parallel_validate_schema(validator, file_names): - num_processors = multiprocessing.cpu_count() - logging.info('Schema validation: %s processors for %s schema validations', num_processors, len(file_names)) - - processor_pool = multiprocessing.Pool(num_processors) - # How to get all the results - result = None - try: - result = processor_pool.map(validator.validate_schema_file, file_names) - except multiprocessing.pool.MaybeEncodingError as error: - pass - return result + if not validator.options.run_serial: + num_processors = multiprocessing.cpu_count() + logging.info('Schema validation: %s processors for %s schema validations', + num_processors, len(file_names)) + + processor_pool = multiprocessing.Pool(num_processors) + # How to get all the results + result = None + try: + result = processor_pool.map(validator.validate_schema_file, file_names) + except multiprocessing.pool.MaybeEncodingError as error: + pass + return result + else: + results = [] + logging.info('check_schemas running serially on %s files!', + len(filenames)) + for file_name in file_names: + results.append(validator.validate_schema_file(file_name)) + return results def main(args): @@ -83,6 +96,8 @@ def main(args): # Todo: use setters to initialize validator validator.schema_base = '.' + validator.options = SchemaArgs(args).getOptions() + if len(args) > 1: schema_base = args[1] else: diff --git a/schema/check_test_output.py b/schema/check_test_output.py index 5d45d676..25153daf 100644 --- a/schema/check_test_output.py +++ b/schema/check_test_output.py @@ -14,6 +14,10 @@ import schema_files from schema_files import ALL_TEST_TYPES +# To get commanlin arguments +sys.path.append('../testdriver') +from ddtargs import schemaArgs + def main(args): logging.config.fileConfig("../logging.conf") diff --git a/testdriver/ddtargs.py b/testdriver/ddtargs.py index b7abf19a..02307b34 100644 --- a/testdriver/ddtargs.py +++ b/testdriver/ddtargs.py @@ -103,6 +103,21 @@ def getOptions(self): return self.options +class schemaArgs(): + def __init__(self, args): + self.parser = argparse.ArgumentParser( + description='Schema check arguments') + + self.parser.add_argument('--run_serial', default=None, + help='Set if execution should be done serially. Parallel is the default.') + self.options = self.parser.parse_args(args) + + + + def getOptions(self): + return self.options + + # Set up arguments common to both testDriver and verifier def setCommonArgs(parser): From c6f5b22f01f14f77eb376715c52d466d0ed22f98 Mon Sep 17 00:00:00 2001 From: sven-oly Date: Tue, 4 Mar 2025 14:34:19 -0800 Subject: [PATCH 02/13] --a --- schema/check_generated_data.py | 5 +++- schema/check_schemas.py | 37 +++++++++++++++++++--------- schema/check_test_output.py | 4 +++ schema/datetime_fmt/test_schema.json | 19 +++++++++++++- testdriver/ddtargs.py | 15 +++++++++++ 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/schema/check_generated_data.py b/schema/check_generated_data.py index df433de8..3daed44d 100644 --- a/schema/check_generated_data.py +++ b/schema/check_generated_data.py @@ -12,6 +12,10 @@ import os.path import sys +# To get commanlin arguments +sys.path.append('../testdriver') +from ddtargs import schemaArgs + import schema_validator from schema_files import ALL_TEST_TYPES @@ -101,6 +105,5 @@ def main(args): logging.info("All %d generated test data files match with schema", schema_count) - if __name__ == "__main__": main(sys.argv) diff --git a/schema/check_schemas.py b/schema/check_schemas.py index c16865cc..223d00e8 100644 --- a/schema/check_schemas.py +++ b/schema/check_schemas.py @@ -15,6 +15,10 @@ import schema_validator from schema_files import ALL_TEST_TYPES +# To get commanlin arguments +sys.path.append('../testdriver') +from ddtargs import schemaArgs + class ValidateSchema: def __init__(self, schema_base='.'): @@ -61,17 +65,26 @@ def save_schema_validation_summary(self, validation_status): def parallel_validate_schema(validator, file_names): - num_processors = multiprocessing.cpu_count() - logging.info('Schema validation: %s processors for %s schema validations', num_processors, len(file_names)) - - processor_pool = multiprocessing.Pool(num_processors) - # How to get all the results - result = None - try: - result = processor_pool.map(validator.validate_schema_file, file_names) - except multiprocessing.pool.MaybeEncodingError as error: - pass - return result + if not validator.options.run_serial: + num_processors = multiprocessing.cpu_count() + logging.info('Schema validation: %s processors for %s schema validations', + num_processors, len(file_names)) + + processor_pool = multiprocessing.Pool(num_processors) + # How to get all the results + result = None + try: + result = processor_pool.map(validator.validate_schema_file, file_names) + except multiprocessing.pool.MaybeEncodingError as error: + pass + return result + else: + results = [] + logging.info('check_schemas running serially on %s files!', + len(filenames)) + for file_name in file_names: + results.append(validator.validate_schema_file(file_name)) + return results def main(args): @@ -83,6 +96,8 @@ def main(args): # Todo: use setters to initialize validator validator.schema_base = '.' + validator.options = SchemaArgs(args).getOptions() + if len(args) > 1: schema_base = args[1] else: diff --git a/schema/check_test_output.py b/schema/check_test_output.py index 5d45d676..25153daf 100644 --- a/schema/check_test_output.py +++ b/schema/check_test_output.py @@ -14,6 +14,10 @@ import schema_files from schema_files import ALL_TEST_TYPES +# To get commanlin arguments +sys.path.append('../testdriver') +from ddtargs import schemaArgs + def main(args): logging.config.fileConfig("../logging.conf") diff --git a/schema/datetime_fmt/test_schema.json b/schema/datetime_fmt/test_schema.json index d18952bb..ea409085 100644 --- a/schema/datetime_fmt/test_schema.json +++ b/schema/datetime_fmt/test_schema.json @@ -39,6 +39,14 @@ "description": "Skeleton for date/time format: https://unicode-org.github.io/icu/userguide/format_parse/datetime/", "type": "string" }, + "datetime_skeleton": { + "description": "Skeleton for date/time format: https://unicode-org.github.io/icu/userguide/format_parse/datetime/", + "type": "string" + }, + "skeleton": { + "description": "Skeleton for date/time format: https://unicode-org.github.io/icu/userguide/format_parse/datetime/", + "type": "string" + }, "options": { "description": "Formatting paramters for date / time, similar to ECMAScript Intl's options bag", "type": "object", @@ -123,7 +131,16 @@ "type": "string", "description": "shortcut for the time style", "enum": ["full", "long", "medium", "short"] - } + }, + "semanticSkeleton": { + "type": "string", + "description": "Semantic skeleton for use >= ICU77" + }, + "semanticSkeletonLength": { + "type": "string", + "description": "what type of semantic skeleton", + "enum": ["long", "medium", "short"] + } } } }, diff --git a/testdriver/ddtargs.py b/testdriver/ddtargs.py index b7abf19a..02307b34 100644 --- a/testdriver/ddtargs.py +++ b/testdriver/ddtargs.py @@ -103,6 +103,21 @@ def getOptions(self): return self.options +class schemaArgs(): + def __init__(self, args): + self.parser = argparse.ArgumentParser( + description='Schema check arguments') + + self.parser.add_argument('--run_serial', default=None, + help='Set if execution should be done serially. Parallel is the default.') + self.options = self.parser.parse_args(args) + + + + def getOptions(self): + return self.options + + # Set up arguments common to both testDriver and verifier def setCommonArgs(parser): From 4a3f16ed19134282247d540a9367f239f578752f Mon Sep 17 00:00:00 2001 From: sven-oly Date: Fri, 29 Aug 2025 10:41:51 -0700 Subject: [PATCH 03/13] Restructuring command line arguments for Schema processing --- schema/check_generated_data.py | 10 ++++++---- schema/check_schemas.py | 3 ++- schema/check_test_output.py | 15 +++++++++------ schema/schema_validator.py | 4 ++++ testdriver/ddtargs.py | 24 ++++++++++++------------ verifier/detail_template.html | 2 +- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/schema/check_generated_data.py b/schema/check_generated_data.py index 06db7720..91711317 100644 --- a/schema/check_generated_data.py +++ b/schema/check_generated_data.py @@ -12,17 +12,19 @@ import os.path import sys -# To get commandline arguments -sys.path.append('../testdriver') -from ddtargs import schemaArgs - import schema_validator from schema_files import ALL_TEST_TYPES +# To get commandline arguments +sys.path.append('../testdriver') +from ddtargs import SchemaArgs + def main(args): logging.config.fileConfig("../logging.conf") + schema_options = SchemaArgs(args).getOptions() + if len(args) <= 1: logging.error('Please specify the path to test data directory') return diff --git a/schema/check_schemas.py b/schema/check_schemas.py index 223d00e8..24538f38 100644 --- a/schema/check_schemas.py +++ b/schema/check_schemas.py @@ -15,9 +15,10 @@ import schema_validator from schema_files import ALL_TEST_TYPES + # To get commanlin arguments sys.path.append('../testdriver') -from ddtargs import schemaArgs +from ddtargs import SchemaArgs class ValidateSchema: diff --git a/schema/check_test_output.py b/schema/check_test_output.py index 25153daf..a8dca8c4 100644 --- a/schema/check_test_output.py +++ b/schema/check_test_output.py @@ -16,17 +16,19 @@ # To get commanlin arguments sys.path.append('../testdriver') -from ddtargs import schemaArgs +from ddtargs import SchemaArgs def main(args): logging.config.fileConfig("../logging.conf") - if len(args) <= 1: - logging.error('Please specify the path to the test output directory') - sys.exit(1) - else: - test_output_path = args[1] + schema_options = SchemaArgs(args).getOptions() + + # file_base + output_path + test_output_path = schema_options.test_output_path + schema_options.output_path + + # TODO: Use this + run_serial = schema_options.run_serial logging.debug('TEST OUTPUT PATH = %s', test_output_path) @@ -47,6 +49,7 @@ def main(args): executor_set.add(os.path.basename(path)) icu_path = os.path.join(test_output_path, '*', 'icu*') + icu_dirs = glob.glob(icu_path) test_output_json_path = os.path.join(test_output_path, '*', 'icu*', '*.json') diff --git a/schema/schema_validator.py b/schema/schema_validator.py index 7c0e1fa6..721713a5 100644 --- a/schema/schema_validator.py +++ b/schema/schema_validator.py @@ -17,6 +17,10 @@ import schema_files from schema_files import SCHEMA_FILE_MAP +sys.path.append('../testdriver') +import datasets as ddt_data +from ddtargs import SchemaArgs + # ?? Move to the initialization ch = logging.StreamHandler() ch.setLevel(logging.INFO) diff --git a/testdriver/ddtargs.py b/testdriver/ddtargs.py index 02307b34..2f56a044 100644 --- a/testdriver/ddtargs.py +++ b/testdriver/ddtargs.py @@ -59,10 +59,6 @@ def __init__(self, args): self.parser.add_argument('--noverify', default=None) # self.parser.add_argument('--custom_verifier', default=None) # - self.parser.add_argument( - '--run_serial', default=None, - help='Set if execution should be done serially. Parallel is the default.') - self.options = self.parser.parse_args(args) def parse(self): @@ -93,9 +89,6 @@ def __init__(self, args): self.parser.add_argument('--test_verifier', help='Flag to run in test mode', default=None) - self.parser.add_argument('--run_serial', default=None, - help='Set if execution should be done serially. Parallel is the default.') - self.options = self.parser.parse_args(args) return @@ -103,16 +96,17 @@ def getOptions(self): return self.options -class schemaArgs(): +class SchemaArgs(): def __init__(self, args): self.parser = argparse.ArgumentParser( description='Schema check arguments') - self.parser.add_argument('--run_serial', default=None, - help='Set if execution should be done serially. Parallel is the default.') - self.options = self.parser.parse_args(args) - + set.parser.add_argment('--schema_base', + help='Where the schemas are based' + ) + setCommonArgs(self.parser) + self.options = self.parser.parse_args(args) def getOptions(self): return self.options @@ -168,6 +162,11 @@ def setCommonArgs(parser): parser.add_argument('--ignore', default=None) + parser.add_argument( + '--run_serial', default=None, + help='Set if processing should be done serially. Parallel is the default.') + + def argsTestData(): tests = [ ['--test_type', 'collation'], @@ -193,6 +192,7 @@ def argsTestData(): ] return tests + def main(args): argparse = DdtArgs() diff --git a/verifier/detail_template.html b/verifier/detail_template.html index a253c35f..d8aa7171 100644 --- a/verifier/detail_template.html +++ b/verifier/detail_template.html @@ -21,7 +21,7 @@ -$test_type with $library_name +$test_type $library_name