Merge pull request #85 from MatBarba/mbarba/test_vs_core

Script to test ORM vs core + fixes

Author: JAlvarezJarreta

src/python/ensembl/core/models.py

@@ -196,12 +196,15 @@ class DNAAlignFeatureAttrib(Base):
         Index("ditag_value_idx", "value", mysql_length=10),
     )
 
-    dna_align_feature_attrib_id: Column = Column(INTEGER(10), primary_key=True)
     dna_align_feature_id: Column = Column(
-        INTEGER(10), ForeignKey("dna_align_feature.dna_align_feature_id"), nullable=False, index=True
+        INTEGER(10),
+        ForeignKey("dna_align_feature.dna_align_feature_id"),
+        nullable=False,
+        index=True,
+        primary_key=True,
     )
-    attrib_type_id: Column = Column(SMALLINT(5), nullable=False)
-    value: Column = Column(Text, nullable=False)
+    attrib_type_id: Column = Column(SMALLINT(5), nullable=False, primary_key=True)
+    value: Column = Column(String(500), nullable=False, primary_key=True)
 
 
 class ExternalDb(Base):
@@ -415,10 +418,11 @@ class RNAproductAttrib(Base):
         Index("rnaproduct_type_val_idx", "attrib_type_id", "value", mysql_length={"value": 10}),
         Index("rnaproduct_value_idx", "value", mysql_length=10),
     )
-    rnaproduct_attrib_id: Column = Column(INTEGER(10), primary_key=True)
-    rnaproduct_id: Column = Column(ForeignKey("rnaproduct.rnaproduct_id"), nullable=False, index=True)
-    attrib_type_id: Column = Column(SMALLINT(5), nullable=False)
-    value: Column = Column(Text, nullable=False)
+    rnaproduct_id: Column = Column(
+        ForeignKey("rnaproduct.rnaproduct_id"), nullable=False, index=True, primary_key=True
+    )
+    attrib_type_id: Column = Column(SMALLINT(5), nullable=False, primary_key=True)
+    value: Column = Column(String(500), nullable=False, primary_key=True)
 
 
 class RnaproductType(Base):
@@ -652,21 +656,22 @@ class GeneAttrib(Base):
         Index("gene_attrib_value_idx", "value", mysql_length=10),
     )
 
-    gene_attrib_id: Column = Column(INTEGER(10), primary_key=True)
     gene_id: Column = Column(
         INTEGER(10),
         ForeignKey("gene.gene_id"),
         nullable=False,
         index=True,
         server_default=text("'0'"),
+        primary_key=True,
     )
     attrib_type_id: Column = Column(
         SMALLINT(5),
         ForeignKey("attrib_type.attrib_type_id"),
         nullable=False,
         server_default=text("'0'"),
+        primary_key=True,
     )
-    value: Column = Column(Text, nullable=False)
+    value: Column = Column(String(500), nullable=False, primary_key=True)
 
 
 class MarkerMapLocation(Base):
@@ -862,21 +867,22 @@ class TranscriptAttrib(Base):
         Index("transcript_attrib_value_idx", "value", mysql_length=10),
     )
 
-    transcript_attrib_id: Column = Column(INTEGER(10), primary_key=True)
     transcript_id: Column = Column(
         INTEGER(10),
         ForeignKey("transcript.transcript_id"),
         nullable=False,
         index=True,
         server_default=text("'0'"),
+        primary_key=True,
     )
     attrib_type_id: Column = Column(
         SMALLINT(5),
         ForeignKey("attrib_type.attrib_type_id"),
         nullable=False,
         server_default=text("'0'"),
+        primary_key=True,
     )
-    value: Column = Column(Text, nullable=False)
+    value: Column = Column(String(500), nullable=False, primary_key=True)
 
 
 class TranscriptSupportingFeature(Base):
@@ -904,7 +910,7 @@ class TranscriptSupportingFeature(Base):
 
 
 class TranslationAttrib(Base):
-    __tablename__ = ("translation_attrib",)
+    __tablename__ = "translation_attrib"
     __table_args__ = (
         Index("translation_attrib_type_val_idx", "attrib_type_id", "value", mysql_length={"value": 10}),
         Index(
@@ -918,19 +924,20 @@ class TranslationAttrib(Base):
         Index("translation_attrib_value_idx", "value", mysql_length=10),
     )
 
-    translation_attrib_id: Column = Column(INTEGER(10), primary_key=True)
     translation_id: Column = Column(
         ForeignKey("translation.translation_id"),
         nullable=False,
         index=True,
         server_default=text("'0'"),
+        primary_key=True,
     )
     attrib_type_id: Column = Column(
         ForeignKey("attrib_type.attrib_type_id"),
         nullable=False,
         server_default=text("'0'"),
+        primary_key=True,
     )
-    value: Column = Column(Text, nullable=False)
+    value: Column = Column(String(500), nullable=False, primary_key=True)
 
 
 class UnmappedObject(Base):
@@ -1850,21 +1857,22 @@ class MiscAttrib(Base):
         Index("misc_attrib_value_idx", "value", mysql_length=10),
     )
 
-    misc_attrib_id: Column = Column(INTEGER(10), primary_key=True)
     misc_feature_id: Column = Column(
         INTEGER(10),
         ForeignKey("misc_feature.misc_feature_id"),
         nullable=False,
         index=True,
         server_default=text("'0'"),
+        primary_key=True,
     )
     attrib_type_id: Column = Column(
         SMALLINT(5),
         ForeignKey("attrib_type.attrib_type_id"),
         nullable=False,
         server_default=text("'0'"),
+        primary_key=True,
     )
-    value: Column = Column(Text, nullable=False)
+    value: Column = Column(String(500), nullable=False, primary_key=True)
 
 
 class OntologyXref(Base):

src/python/tests/core/compare_core_model.py

@@ -0,0 +1,79 @@
+# See the NOTICE file distributed with this work for additional information
+# regarding copyright ownership.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Check the current ensembl-py core model against a core created from Ensembl SQL.
+This script gets one row for each table in the ORM to check that SQLAlchemy can correctly query the table.
+If not, it will show the `OperationalError` exception to explain what is wrong in the ORM.
+
+Use this script to check the ORM (and fix it if needed).
+"""
+
+import logging
+
+from ensembl.core.models import Base
+from ensembl.utils.database import DBConnection
+from ensembl.utils.argparse import ArgumentParser
+from ensembl.utils.logging import init_logging_with_args
+from sqlalchemy import select
+from sqlalchemy.exc import NoResultFound, MultipleResultsFound, OperationalError, ProgrammingError
+from sqlalchemy.orm import Session
+
+
+def check_tables(session: Session, only_table: str = "") -> None:
+    """Load data from a core using the ORM to check for any discrepancies in the definitions.
+
+    Args:
+        session: SQLAlchemy session.
+        only_table: Only check this one table instead of all of the tables defined in the ORM.
+    """
+    success = []
+    errors = []
+    for table_name, table in Base.metadata.tables.items():
+        if isinstance(table_name, tuple):
+            table_name = table_name[0]
+        if only_table and table_name != only_table:
+            continue
+        logging.debug(f"Check table {table_name}")
+        stmt = select(table)
+        try:
+            session.execute(stmt).one()
+            success.append(table_name)
+        except (NoResultFound, MultipleResultsFound):
+            success.append(table_name)
+        except (OperationalError, ProgrammingError) as err:
+            # Show the problematic query and continue
+            logging.warning(f"{table_name}: {err}")
+            errors.append(table_name)
+
+    logging.info(f"{len(success)} tables successfully queried with the ORM")
+    if errors:
+        logging.warning(f"{len(errors)} tables failed to be queried with the ORM: {', '.join(errors)}")
+    else:
+        logging.info("No errors found")
+
+
+def main() -> None:
+    """Main script entry-point."""
+    parser = ArgumentParser(description=__doc__)
+    parser.add_server_arguments(include_database=True, help="Ensembl MySQL core database")
+    parser.add_argument("--table", type=str, help="Test this one table only")
+    parser.add_log_arguments(add_log_file=True)
+    args = parser.parse_args()
+    init_logging_with_args(args)
+    dbc = DBConnection(args.url, reflect=False)
+    with dbc.session_scope() as session:
+        check_tables(session, only_table=args.table)
+
+
+if __name__ == "__main__":
+    main()