Merge branch 'main' into sqlalchemy2

Author: Iain-S

docs/source/_static/config_schema.html

@@ -1,5 +1,11 @@
 Configuration
 -------------
 
+SqlSynthGen is configured using a YAML file, which is passed to several commands with the ``--config`` option.
+Throughout the docs, we will refer to this file as ``config.yaml`` but it can be called anything (the exception being that there will be a naming conflict if you have a vocabulary table called ``config``).
+
+Below, we see the schema for the configuration file.
+Note that our config file format includes a section of SmartNoise SQL metadata, which is explained more fully `here <https://docs.smartnoise.org/sql/metadata.html#yaml-format>`_.
+
 .. raw:: html
    :file: _static/config_schema.html

docs/source/configuration.rst

@@ -6,56 +6,94 @@
   "additionalProperties": false,
   "properties": {
     "use-asyncio": {
+      "description": "Run source-statistics queries using asyncpg.",
       "type": "boolean"
     },
     "row_generators_module": {
+      "description": "The name of a local Python module of row generators (excluding .py).",
       "type": "string"
     },
     "story_generators_module": {
+      "description": "The name of a local Python module of story generators (excluding .py).",
       "type": "string"
     },
-
     "src-stats": {
+      "description": "An array of source statistics queries.",
       "type": "array",
       "items": {
         "additionalProperties": false,
         "required": ["name", "query"],
         "properties": {
           "name": {
+            "description": "A name for the query, which will be used in the stats file.",
             "type": "string"
           },
           "query": {
+            "description": "A SQL query.",
             "type": "string"
           },
           "dp-query": {
+            "description": "A SmartNoise SQL query.",
             "type": "string"
           },
           "epsilon": {
+            "description": "The differential privacy epsilon value for the DP query.",
             "type": "number"
           },
           "delta": {
+            "description": "The differential privacy delta value for the DP query.",
             "type": "number"
           },
-
           "snsql-metadata": {
+            "description": "See https://docs.smartnoise.org/sql/metadata.html#yaml-format.",
             "type": "object",
             "properties": {
+              "max_ids": {
+                "type": "integer"
+              },
+              "row_privacy": {
+                "type": "boolean"
+              },
+              "sample_max_ids": {
+                "type": "boolean"
+              },
               "censor_dims": {
                 "type": "boolean"
+              },
+              "clamp_counts": {
+                "type": "boolean"
+              },
+              "clamp_columns": {
+                "type": "boolean"
+              },
+              "use_dpsu": {
+                "type": "boolean"
               }
             },
             "patternProperties": {
-              "^(?!censor_dims).*$": {
+              "^(?!(max_ids|row_privacy|sample_max_ids|censor_dims|clamp_counts|clamp_columns|use_dpsu)).*$": {
                 "type": "object",
+                "additionalProperties": false,
+                "required": ["type"],
                 "properties": {
-                  "name": {
-                    "type": "string"
-                  },
                   "type": {
                     "type": "string"
                   },
                   "private_id": {
                     "type": "boolean"
+                  },
+                  "lower": {
+                    "type": "number"
+                  },
+                  "upper": {
+                    "type": "number"
+                  },
+                  "nullable": {
+                    "type": "boolean"
+                  },
+                  "missing_value": {},
+                  "sensitivity": {
+                    "type": "number"
                   }
                 }
               }
@@ -66,60 +104,76 @@
     },
 
     "story_generators": {
+      "description": "An array of story generators.",
       "type": "array",
       "items": {
         "type": "object",
         "additionalProperties": false,
         "required": ["name", "num_stories_per_pass"],
         "properties": {
           "name": {
+            "description": "The full name of a story generator (e.g. my_story_generators.short_story).",
             "type": "string"
           },
           "args": {
+            "description": "Positional arguments to pass to the story generator.",
             "type": "array"
           },
           "kwargs": {
+            "description": "Keyword arguments to pass to the story generator.",
             "type": "object"
           },
           "num_stories_per_pass": {
+            "description": "The number of times to call the story generator per pass.",
             "type": "integer"
           }
         }
       }
     },
 
     "max-unique-constraint-tries": {
+      "description": "The maximum number of tries to respect a uniqueness constraint.",
       "type": "integer"
     },
 
     "tables": {
+      "description": "Table configurations.",
       "type": "object",
       "patternProperties": {
         ".*": {
+          "description": "A table configuration.",
           "additionalProperties": false,
           "type": "object",
           "properties": {
             "vocabulary_table": {
+              "description": "Whether to export the table data.",
               "type": "boolean"
             },
             "num_rows_per_pass": {
+              "description": "The number of rows to generate per pass.",
               "type": "integer"
             },
             "row_generators": {
+              "description": "An array of row generators to create column values.",
               "type": "array",
               "items": {
                 "type": "object",
+                "required": ["name", "columns_assigned"],
                 "properties": {
                   "name": {
+                    "description": "The name of a (built-in or custom) function (e.g. max or my_row_generators.my_gen).",
                     "type": "string"
                   },
                   "args": {
+                    "description": "Positional arguments to pass to the function.",
                     "type": "array"
                   },
                   "kwargs": {
+                    "description": "Keyword arguments to pass to the function.",
                     "type": "object"
                   },
                   "columns_assigned": {
+                    "description": "One or more columns to assign the return value to.",
                     "type": ["array", "string"],
                     "items": {
                       "type": "string"

sqlsynthgen/json_schemas/config_schema.json

@@ -16,16 +16,13 @@
 from sqlsynthgen.make import make_src_stats, make_table_generators, make_tables_file
 from sqlsynthgen.remove import remove_db_data, remove_db_tables, remove_db_vocab
 from sqlsynthgen.settings import Settings, get_settings
-from sqlsynthgen.utils import import_file, read_yaml_file
+from sqlsynthgen.utils import CONFIG_SCHEMA_PATH, import_file, read_config_file
 
 # pylint: disable=too-many-arguments
 
 ORM_FILENAME: Final[str] = "orm.py"
 SSG_FILENAME: Final[str] = "ssg.py"
 STATS_FILENAME: Final[str] = "src-stats.yaml"
-CONFIG_SCHEMA_PATH: Final[Path] = (
-    Path(__file__).parent / "json_schemas/config_schema.json"
-)
 
 app = Typer(no_args_is_help=True)
 
@@ -182,7 +179,7 @@ def make_generators(
     _require_src_db_dsn(settings)
 
     orm_module: ModuleType = import_file(orm_file)
-    generator_config = read_yaml_file(config_file) if config_file is not None else {}
+    generator_config = read_config_file(config_file) if config_file is not None else {}
     result: str = make_table_generators(
         orm_module, generator_config, stats_file, overwrite_files=force
     )
@@ -214,7 +211,7 @@ def make_stats(
     if not force:
         _check_file_non_existence(stats_file_path)
 
-    config = read_yaml_file(config_file) if config_file is not None else {}
+    config = read_config_file(config_file) if config_file is not None else {}
 
     settings = get_settings()
     src_dsn: str = _require_src_db_dsn(settings)

sqlsynthgen/main.py

@@ -1,20 +1,43 @@
 """Utility functions."""
+import json
 import os
 import sys
 from importlib import import_module
 from pathlib import Path
 from types import ModuleType
-from typing import Any, Optional, Union
+from typing import Any, Final, Optional, Union
 
 import yaml
+from jsonschema.exceptions import ValidationError
+from jsonschema.validators import validate
 from sqlalchemy import create_engine, event, select
 from sqlalchemy.ext.asyncio import create_async_engine
 
+CONFIG_SCHEMA_PATH: Final[Path] = (
+    Path(__file__).parent / "json_schemas/config_schema.json"
+)
 
-def read_yaml_file(path: str) -> Any:
-    """Read a yaml file in to dictionary, given a path."""
+
+def read_config_file(path: str) -> dict:
+    """Read a config file, warning if it is invalid.
+
+    Args:
+        path: The path to a YAML-format config file.
+
+    Returns:
+        The config file as a dictionary.
+    """
     with open(path, "r", encoding="utf8") as f:
         config = yaml.safe_load(f)
+
+    assert isinstance(config, dict)
+
+    schema_config = json.loads(CONFIG_SCHEMA_PATH.read_text(encoding="UTF-8"))
+    try:
+        validate(config, schema_config)
+    except ValidationError as e:
+        print("The config file is invalid:", e.message)
+
     return config
 
 

sqlsynthgen/utils.py

@@ -25,13 +25,22 @@ src-stats:
     snsql-metadata:
       # You may well want censor_dims to be on, but we turn it off for the
       # tests to silence a smartnoise-sql nag warning.
+      max_ids: 10
+      row_privacy: False
+      sample_max_ids: True
       censor_dims: False
+      clamp_counts: False
+      clamp_columns: False
+      use_dpsu: False
       person_id:
-        name: person_id
         type: int
         private_id: True
+        lower: 10
+        upper: 100
+        nullable: True
+        sensitivity: 1
+        # missing_value: breaks things
       research_opt_out:
-        name: research_opt_out
         type: boolean
         private_id: False
 

tests/examples/example_config.yaml

@@ -1,14 +1,21 @@
 """Tests for the utils module."""
 import os
 import sys
+from io import StringIO
 from pathlib import Path
+from unittest.mock import patch
 
 from pydantic import PostgresDsn
 from pydantic.tools import parse_obj_as
 from sqlalchemy import Column, Integer, create_engine, insert
 from sqlalchemy.orm import declarative_base
 
-from sqlsynthgen.utils import create_db_engine, download_table, import_file
+from sqlsynthgen.utils import (
+    create_db_engine,
+    download_table,
+    import_file,
+    read_config_file,
+)
 from tests.utils import RequiresDBTestCase, SSGTestCase, run_psql
 
 # pylint: disable=invalid-name
@@ -116,3 +123,14 @@ def test_connect_async(self) -> None:
 
         # With schema
         create_db_engine(self.dsn, schema_name="public", use_asyncio=True)
+
+
+class TestReadConfig(SSGTestCase):
+    """Tests for the read_config_file function."""
+
+    def test_warns_of_invalid_config(self) -> None:
+        """Test that we get a warning if the config is invalid."""
+        with patch("sys.stdout", new=StringIO()) as mock_stdout:
+            read_config_file("tests/examples/invalid_config.yaml")
+
+        self.assertIn("The config file is invalid:", mock_stdout.getvalue())