Updates for running the testsuite using realtime database

* Fix python3 MultiOrderedDict.keys() issue in astconfigparser.py.
* Updated realtime_converter.py to be compatible with SQLAlchemy 2.
* Added requirements-realtime.txt to install SQLAlchemy, alembic and
  psycopg2.
* Updated setupVenv.sh to install from requirements-realtime.txt
  if the `--realtime` argument was passed in.

Author: gtjoseph

lib/python/asterisk/astconfigparser.py

@@ -106,7 +106,7 @@ def keys(self, self_only=False):
         keys from this section's defaults and templates are not
         included in the returned value
         """
-        res = MultiOrderedDict.keys(self)
+        res = list(MultiOrderedDict.keys(self))
         if self_only:
             return res
 
@@ -119,7 +119,7 @@ def keys(self, self_only=False):
             for key in d.keys():
                 if key not in res:
                     res.append(key)
-        return list(res.keys())
+        return res
 
     def add_defaults(self, defaults):
         """

lib/python/asterisk/realtime_converter.py

@@ -151,7 +151,7 @@ def write_db(self, config_dir, meta, engine, conn, test_object):
                                 "type {0}".format(obj_type))
                     continue
                 table = Table(self.sections[sorcery_section][obj_type], meta,
-                              autoload=True, autoload_with=engine)
+                              autoload_with=engine)
                 vals = {'id': title}
                 for key in section.keys():
                     key_name = test_object.ast[0].configuration_replace_string(key)
@@ -190,10 +190,9 @@ def cleanup_db(self, meta, engine, conn):
         conn: sqlalchemy Connection to database
         """
         for table_name in self.tables:
-            table = Table(table_name, meta, autoload=True,
-                          autoload_with=engine)
+            table = Table(table_name, meta, autoload_with=engine)
             conn.execute(table.delete())
-
+        conn.commit()
 
 class RealtimeConverter(object):
     """Pluggable module used to convert configuration files to database data.
@@ -302,9 +301,14 @@ def write_modules_conf(self):
 
     def write_db(self, test_object):
         """Tell converters to write database information"""
-        for realtime_file in REALTIME_FILE_REGISTRY:
-            realtime_file.write_db(self.config_dir, self.meta, self.engine,
-                                   self.conn, test_object)
+        try:
+            for realtime_file in REALTIME_FILE_REGISTRY:
+                realtime_file.write_db(self.config_dir, self.meta, self.engine,
+                                       self.conn, test_object)
+        except:
+            self.conn.rollback()
+            raise
+        self.conn.commit()
 
     def cleanup(self, result):
         """Cleanup information after test has completed.

requirements-realtime.txt

@@ -0,0 +1,5 @@
+# These packages are required if running the testsuite with
+# the realtime_converter
+SQLAlchemy==2.0.30
+alembic==1.13.1
+psycopg2==2.9.9

setupVenv.sh

@@ -1,22 +1,33 @@
 #!/usr/bin/env bash
 set -e
 
+REALTIME=false
+declare -a VENV_ARGS
+
+for a in "$@" ; do
+	if [ "$a" == "--realtime" ] ; then
+		REALTIME=true
+	else
+		VENV_ARGS+=( "$a" )
+	fi
+done
+
 function do_pip_setup {
 	python3 -m pip install --upgrade pip
 	python3 -m pip install wheel setuptools build
 	python3 -m pip install -r ./requirements.txt
 	python3 -m pip install -r ./extras.txt
-	md5sum requirements.txt extras.txt > $1/checksums
+	$REALTIME && python3 -m pip install -r ./requirements-realtime.txt
+	md5sum requirements.txt extras.txt requirements-realtime.txt > $1/checksums
 }
 
-
 if [[ "$VIRTUAL_ENV" != "" ]]
 then
 	echo "Detected activated virtual environment:" $VIRTUAL_ENV
 	echo "Skipping creation of new environment, configuring"
 	do_pip_setup $VIRTUAL_ENV
 else
-	python3 -m venv $@ .venv
+	python3 -m venv ${VENV_ARGS[@]} .venv
 	source .venv/bin/activate
 	echo "Activated virtual environment:" $VIRTUAL_ENV
 	if [[ "$VIRTUAL_ENV" != "" ]]