diff --git a/.github/workflows/indigo-ci.yaml b/.github/workflows/indigo-ci.yaml index 548cd13381..5417ba2668 100644 --- a/.github/workflows/indigo-ci.yaml +++ b/.github/workflows/indigo-ci.yaml @@ -1086,6 +1086,101 @@ jobs: name: bingo-oracle-windows-msvc-x86_64 path: dist/bingo-oracle*.zip + test_bingo_oracle_linux_x86_64: + runs-on: ubuntu-latest + needs: [build_bingo_oracle_linux_x86_64] + services: + oracle: + image: gvenzl/oracle-xe:21-slim + env: + ORACLE_PASSWORD: password + ports: + - 1521:1521 + options: >- + --health-cmd healthcheck.sh + --health-interval 30s + --health-timeout 10s + --health-retries 10 + --health-start-period 120s + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: false + fetch-depth: 500 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Git fetch tags + run: | + git config --global --add safe.directory '*' + git fetch --tags -f + - name: Install Oracle Instant Client + run: | + sudo apt-get update + sudo apt-get install -y libaio1t64 || sudo apt-get install -y libaio1 + if [ ! -e /usr/lib/x86_64-linux-gnu/libaio.so.1 ] && [ -e /usr/lib/x86_64-linux-gnu/libaio.so.1t64 ]; then + sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 + fi + wget -q https://download.oracle.com/otn_software/linux/instantclient/2350000/instantclient-basic-linux.x64-23.5.0.24.07.zip + wget -q https://download.oracle.com/otn_software/linux/instantclient/2350000/instantclient-sqlplus-linux.x64-23.5.0.24.07.zip + sudo mkdir -p /opt/oracle + sudo unzip -o instantclient-basic-linux.x64-23.5.0.24.07.zip -d /opt/oracle + sudo unzip -o instantclient-sqlplus-linux.x64-23.5.0.24.07.zip -d /opt/oracle + echo "/opt/oracle/instantclient_23_5" | sudo tee /etc/ld.so.conf.d/oracle.conf + sudo ldconfig + echo "/opt/oracle/instantclient_23_5" >> $GITHUB_PATH + - name: Download bingo-oracle artifact + uses: actions/download-artifact@v4 + with: + name: bingo-oracle-linux-x86_64 + path: dist + - name: Install bingo-oracle + env: + ORACLE_HOME: /opt/oracle/instantclient_23_5 + LD_LIBRARY_PATH: /opt/oracle/instantclient_23_5 + run: | + export PATH="/opt/oracle/instantclient_23_5:$PATH" + mkdir -p /opt/bingo-oracle && cd /opt/bingo-oracle + tar -xzf $GITHUB_WORKSPACE/dist/bingo-oracle*.tgz --strip-components=1 + sh ./bingo-oracle-install.sh \ + -libdir /opt/oracle/instantclient_23_5 \ + -dbaname system \ + -dbapass password \ + -instance localhost:1521/XEPDB1 \ + -bingoname bingo \ + -bingopass bingo \ + -y + - name: Create test schema user + env: + LD_LIBRARY_PATH: /opt/oracle/instantclient_23_5 + run: | + echo " + CREATE USER test IDENTIFIED BY test DEFAULT TABLESPACE bingo; + GRANT CONNECT TO test; + GRANT CREATE TABLE TO test; + GRANT CREATE SESSION TO test; + GRANT UNLIMITED TABLESPACE TO test; + GRANT EXECUTE ON bingo.MangoPackage TO test; + GRANT EXECUTE ON bingo.RingoPackage TO test; + EXIT; + " | sqlplus system/password@localhost:1521/XEPDB1 + - name: Install dev dependencies + run: | + pip install -r bingo/tests/requirements.txt --break-system-packages + - name: Run Bingo Oracle tests + run: | + pytest -s --tb=no --db oracle --junit-xml=junit_report.xml + working-directory: bingo/tests + - name: Publish Test Report + if: always() + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'bingo/tests/junit_report.xml' + github_token: ${{ secrets.GITHUB_TOKEN }} + check_name: "bingo_oracle_test_report" + build_bingo_postgres_linux_x86_64: strategy: fail-fast: false diff --git a/bingo/tests/conftest.py b/bingo/tests/conftest.py index b8c957df64..535ef80b9e 100644 --- a/bingo/tests/conftest.py +++ b/bingo/tests/conftest.py @@ -14,6 +14,7 @@ ) from .dbc.BingoNoSQL import BingoNoSQL from .dbc.PostgresSQL import Postgres +from .dbc.OracleDB import Oracle from .helpers import get_bingo_meta, get_query_entities from .logger import logger @@ -67,7 +68,10 @@ def db(request, indigo): db = BingoElastic(indigo, index_name) db.import_data(meta["import_no_sql"], data_type) elif db_str == DB_ORACLE: - pass + db = Oracle() + ora_tables = db.create_data_tables(meta["tables"]) + db.import_data(import_meta=meta["import"]) + db.create_indices(meta["indices"]) elif db_str == DB_MSSQL: pass yield db @@ -82,6 +86,10 @@ def db(request, indigo): db.delete_base() elif db_str == DB_BINGO_ELASTIC: db.drop() + elif db_str == DB_ORACLE: + for table in ora_tables: + logger.info(f"Dropping Oracle table {table}") + table.drop(db.engine) logger.info(f"===== Finish of testing {function} =====") diff --git a/bingo/tests/db_config.ini b/bingo/tests/db_config.ini index b2528c20de..57032838cf 100644 --- a/bingo/tests/db_config.ini +++ b/bingo/tests/db_config.ini @@ -13,6 +13,13 @@ password=password db_name=bingo_nosql_db db_dir=../data +[oracle] +host=localhost +port=1521 +database=XEPDB1 +user=test +password=test + [bingo-elastic] host=localhost port=9200 \ No newline at end of file diff --git a/bingo/tests/dbc/OracleDB.py b/bingo/tests/dbc/OracleDB.py new file mode 100644 index 0000000000..9fe559c41f --- /dev/null +++ b/bingo/tests/dbc/OracleDB.py @@ -0,0 +1,303 @@ +from os import path + +import sqlalchemy as sa +from indigo import IndigoObject +from sqlalchemy import text +from sqlalchemy.engine import create_engine +from sqlalchemy.exc import DatabaseError, InternalError, ProgrammingError +from sqlalchemy.orm.session import sessionmaker + +from ..constants import ( + DB_ORACLE, + IMPORT_FUNCTION_MAP, + TARGET_TABLES_MAP, +) +from ..logger import logger +from .base import SQLAdapter + +MATCHING_SEARCH_QUERY = ( + "SELECT id, 1 from {test_schema}.{table_name} " + "WHERE {bingo_schema}.{function}(data, :query_entity{params_clause})=1 " + "ORDER BY ID ASC" +) + + +class Oracle(SQLAdapter): + dbms = DB_ORACLE + + def __init__(self): + SQLAdapter.__init__(self) + logger.debug(f"Opening connection to {self.dbms}") + self._engine = create_engine(self.conn_string) + session = sessionmaker(bind=self._engine) + session.configure( + bind=self._engine, + autocommit=False, + autoflush=False, + ) + self._session = session() + self._session.dialect = self._engine.dialect + self._connect = self._engine.connect() + + def _execute_query(self, query, entity, table_name, options): + query = query.format( + test_schema=self.test_schema, + bingo_schema=self.bingo_schema, + table_name=table_name, + options=options, + ) + rows = None + try: + result = self._connect.execute( + text(query), {"query_entity": entity.rawData()} + ) + if not result.closed: + rows = result.fetchall() + except (DatabaseError, InternalError, ProgrammingError, Exception) as e: + errors_start_with = [ + "ORA-", + "bingo:", + "(oracledb.exceptions.", + ] + return self._select_error_text(e, errors_start_with, "\n") + return rows + + def create_data_tables(self, tables): + created_tables = [] + sa_meta = sa.MetaData() + for table in tables: + logger.debug(f"Creating {self.dbms} table {table}") + created_tables.append( + sa.Table( + table, + sa_meta, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "data", sa.Text, nullable=True + ), + schema=self.test_schema, + ) + ) + sa_meta.create_all(self.engine) + return created_tables + + def import_data(self, import_meta, other_columns=""): + for table, import_path in import_meta.items(): + import_path_ext = path.splitext(import_path)[1] + function = IMPORT_FUNCTION_MAP.get(import_path_ext) + logger.debug( + f"Importing data to table {table} from src: {import_path}" + ) + for item in function(import_path): + query = ( + "INSERT INTO {test_schema}.{table}(data) " + "VALUES (:item_data)" + ) + query = query.format( + test_schema=self.test_schema, + table=table, + ) + connect = self._connect + t = connect.begin() + connect.execute( + text(query), + {"item_data": item.rawData()}, + ) + t.commit() + + def create_indices(self, tables): + for table in tables: + logger.debug( + f"Creating index {self.test_schema}_{table}_idx" + ) + dml_query = ( + "CREATE INDEX {test_schema}_{table}_idx ON " + "{test_schema}.{table}(data) " + "INDEXTYPE IS {bingo_schema}.MoleculeIndex" + ) + dml_query = dml_query.format( + test_schema=self.test_schema, + table=table, + bingo_schema=self.bingo_schema, + ) + self._execute_dml_query(dml_query) + + def query_row( + self, query: str, entity: IndigoObject, table_name="", options="" + ): + """Execute SQL query and return single row""" + try: + rows = self._execute_query(query, entity, table_name, options) + if rows: + return rows[0][0] + return rows + except Exception as e: + return e + + def query_rows( + self, query: str, entity: IndigoObject, table_name="", options="" + ): + """Execute SQL query and return a list of rows""" + try: + rows = self._execute_query(query, entity, table_name, options) + if type(rows) is list: + rows = [row[0] for row in rows] + + return rows + except Exception as e: + return e + + def checkmolecule(self, molecule): + query_sql = ( + "SELECT {bingo_schema}.CheckMolecule(:query_entity) FROM dual" + ) + return self.query_row(query_sql, molecule) + + def cml(self, molecule): + query_sql = ( + "SELECT {bingo_schema}.CML(:query_entity) FROM dual" + ) + return self.query_row(query_sql, molecule) + + def compactmolecule(self, molecule): + query_sql = ( + "SELECT dbms_crypto.hash(" + "{bingo_schema}.CompactMolecule(:query_entity, 0), 2) FROM dual" + ) + return self.query_row(query_sql, molecule) + + def fingerprint(self, molecule, options): + query_sql = ( + "SELECT dbms_crypto.hash(" + "{bingo_schema}.Fingerprint(:query_entity, '{options}'), 2) " + "FROM dual" + ) + return self.query_row(query_sql, molecule, options=options) + + def gross(self, molecule): + query_sql = ( + "SELECT {bingo_schema}.Gross(:query_entity) FROM dual" + ) + return self.query_row(query_sql, molecule) + + def inchi(self, molecule, options="", inchikey=False): + query_sql = ( + "SELECT {bingo_schema}.InChI(:query_entity, '{options}') FROM dual" + ) + if inchikey: + query_sql = ( + "SELECT {bingo_schema}.InChIKey(" + "{bingo_schema}.InChI(:query_entity, ' ')) FROM dual" + ) + return self.query_row(query_sql, molecule, options=options) + + def mass(self, molecule, options): + query_sql = ( + "SELECT {bingo_schema}.Mass(:query_entity, '{options}') FROM dual" + ) + return self.query_row(query_sql, molecule, options=options) + + def similarity(self, molecule, target_function, sim_type, options=""): + table_name = TARGET_TABLES_MAP.get(target_function) + min_sim, max_sim = options.split(", ") + query_sql = ( + "SELECT id, {bingo_schema}.Sim(data, :query_entity, " + "'{sim_type}') FROM " + "{test_schema}.{table_name} WHERE " + "{bingo_schema}.Sim(data, :query_entity, '{sim_type}') " + "BETWEEN {min_sim} AND {max_sim} ORDER BY id ASC" + ) + query_sql = query_sql.replace("{sim_type}", sim_type) + query_sql = query_sql.replace("{min_sim}", min_sim) + query_sql = query_sql.replace("{max_sim}", max_sim) + return self.query_rows(query_sql, molecule, table_name, options) + + def exact(self, molecule, target_function, options=""): + params_clause = f", '{options}'" if options else "" + query_sql = MATCHING_SEARCH_QUERY.replace( + "{function}", "Exact" + ).replace("{params_clause}", params_clause) + table_name = TARGET_TABLES_MAP.get(target_function) + return self.query_rows(query_sql, molecule, table_name, options) + + def substructure(self, molecule, target_function, options=""): + params_clause = f", '{options}'" if options else "" + query_sql = MATCHING_SEARCH_QUERY.replace( + "{function}", "Sub" + ).replace("{params_clause}", params_clause) + table_name = TARGET_TABLES_MAP.get(target_function) + return self.query_rows(query_sql, molecule, table_name, options) + + def smarts(self, molecule, target_function, options=""): + query_sql = ( + "SELECT id, 1 from {test_schema}.{table_name} " + "WHERE {bingo_schema}.Smarts(data, :query_entity)=1 " + "ORDER BY ID ASC" + ) + table_name = TARGET_TABLES_MAP.get(target_function) + return self.query_rows(query_sql, molecule, table_name, options) + + def aam(self, reaction, options): + query_sql = ( + "SELECT {bingo_schema}.AutoAAM(:query_entity, '{options}') " + "FROM dual" + ) + return self.query_row(query_sql, reaction, options=options) + + def checkreaction(self, reaction, options=""): + query_sql = ( + "SELECT {bingo_schema}.CheckReaction(:query_entity) FROM dual" + ) + return self.query_row(query_sql, reaction, options=options) + + def compactreaction(self, reaction, options="0"): + query_sql = ( + "SELECT dbms_crypto.hash(" + "{bingo_schema}.CompactReaction(:query_entity, 0), 2) FROM dual" + ) + return self.query_row(query_sql, reaction, options=options) + + def rcml(self, reaction, options=""): + query_sql = ( + "SELECT {bingo_schema}.RCML(:query_entity) FROM dual" + ) + return self.query_row(query_sql, reaction, options=options) + + def rfingerprint(self, reaction, options=""): + query_sql = ( + "SELECT dbms_crypto.hash(" + "{bingo_schema}.RFingerprint(:query_entity, '{options}'), 2) " + "FROM dual" + ) + return self.query_row(query_sql, reaction, options=options) + + def rsmiles(self, reaction, options=""): + query_sql = ( + "SELECT {bingo_schema}.RSMILES(:query_entity) FROM dual" + ) + return self.query_row(query_sql, reaction, options=options) + + def rexact(self, reaction, target_function, options=""): + params_clause = f", '{options}'" if options else "" + query_sql = MATCHING_SEARCH_QUERY.replace( + "{function}", "RExact" + ).replace("{params_clause}", params_clause) + table_name = TARGET_TABLES_MAP.get(target_function) + return self.query_rows(query_sql, reaction, table_name, options) + + def rsmarts(self, reaction, target_function, options=""): + query_sql = ( + "SELECT id, 1 from {test_schema}.{table_name} " + "WHERE {bingo_schema}.RSmarts(data, :query_entity)=1 " + "ORDER BY ID ASC" + ) + table_name = TARGET_TABLES_MAP.get(target_function) + return self.query_rows(query_sql, reaction, table_name, options) + + def rsubstructure(self, reaction, target_function, options=""): + params_clause = f", '{options}'" if options else "" + query_sql = MATCHING_SEARCH_QUERY.replace( + "{function}", "RSub" + ).replace("{params_clause}", params_clause) + table_name = TARGET_TABLES_MAP.get(target_function) + return self.query_rows(query_sql, reaction, table_name, options) diff --git a/bingo/tests/dbc/base.py b/bingo/tests/dbc/base.py index 0be8e6197a..f07700b123 100644 --- a/bingo/tests/dbc/base.py +++ b/bingo/tests/dbc/base.py @@ -1,5 +1,6 @@ from abc import abstractmethod from configparser import ConfigParser +import os from os import path from os.path import abspath, join from typing import Dict, List @@ -37,13 +38,19 @@ def get_config(): "test_schema": parser.get("common", "test_schema"), }, DB_POSTGRES: { - "host": parser.get(DB_POSTGRES, "host"), + "host": os.environ.get("DB_POSTGRES_HOST", parser.get(DB_POSTGRES, "host")), "port": parser.get(DB_POSTGRES, "port"), "database": parser.get(DB_POSTGRES, "database"), "user": parser.get(DB_POSTGRES, "user"), "password": parser.get(DB_POSTGRES, "password"), }, - DB_ORACLE: None, + DB_ORACLE: { + "host": os.environ.get("DB_ORACLE_HOST", parser.get(DB_ORACLE, "host")), + "port": parser.get(DB_ORACLE, "port"), + "database": parser.get(DB_ORACLE, "database"), + "user": parser.get(DB_ORACLE, "user"), + "password": parser.get(DB_ORACLE, "password"), + }, DB_MSSQL: None, DB_BINGO: { "db_name": parser.get(DB_BINGO, "db_name"), @@ -161,7 +168,7 @@ def conn_string(self): if self.dbms == "postgres": dialect, driver = "postgresql", "psycopg2" if self.dbms == "oracle": - pass + dialect, driver = "oracle", "oracledb" if self.dbms == "mssql": pass diff --git a/bingo/tests/requirements.txt b/bingo/tests/requirements.txt index b8484b107e..abe768c122 100644 --- a/bingo/tests/requirements.txt +++ b/bingo/tests/requirements.txt @@ -1,5 +1,6 @@ elasticsearch==7.10.1 epam.indigo +oracledb==3.4.2 psycopg2-binary==2.9.3 pytest==6.2.5 SQLAlchemy==1.3.22