Add support for OPENXML with WITH clause (#76)

OPENXML provides a rowset view over an XML document. Since OPENXML is a rowset provider and it returns a set of rows, we can use OPENXML in the FROM clause of a T-SQL statement just as we can use any other table, view, or table-valued function. The WITH clause in OPENXML provides a rowset format (and additional mapping information as required) by using either SchemaDeclaration or specifying an existing TableName.
Currently Babelfish does not have support for openxml. So the primary objective is to add support for OPENXML with WITH clause.
The syntax of openxml is as follows :

OPENXML ( idoc int [ in ] , rowpattern nvarchar [ in ], [ flags byte [ in ] ] )
    [ WITH ( SchemaDeclaration | TableName ) ]
Arguments

idoc
The document handle of the internal representation of an XML document. The internal representation of an XML document is created by calling sp_xml_preparedocument.
rowpattern
The XPath pattern used to identify the nodes to be processed as rows. The nodes come from the XML document whose handle is passed in the idoc parameter.
flags
Indicates the mapping used between the XML data and the relational rowset, and how the spill-over column is filled. flags is an optional input parameter, and can be one of the following values.
The WITH clause provides a rowset format (and additional mapping information as required) by using either SchemaDeclaration or specifying an existing TableName. If the optional WITH clause isn't specified, the results are returned in an edge table format. Edge tables represent the fine-grained XML document structure (such as element/attribute names, the document hierarchy, the namespaces, PIs, and so on) in a single table.

BABEL-3635
Engine PR : amazon-aurora/postgresql_modified_for_babelfish#133

Signed-off-by: Harsh Dubey [email protected]

Author: harshdubey166

contrib/babelfishpg_tsql/sql/sys_functions.sql

@@ -4029,6 +4029,16 @@ CREATE OR REPLACE FUNCTION sys.openjson_with(json_string text, path text, VARIAD
 RETURNS SETOF RECORD
 AS 'babelfishpg_tsql', 'tsql_openjson_with' LANGUAGE C STRICT IMMUTABLE PARALLEL SAFE;
 
+CREATE OR REPLACE FUNCTION sys.tsql_openxml_get_xmldoc(int)
+RETURNS xml
+AS 'babelfishpg_tsql', 'tsql_openxml_get_xmldoc'
+LANGUAGE C STRICT;
+
+CREATE OR REPLACE FUNCTION sys.tsql_openxml_get_colpattern(text,int)
+RETURNS sys.nvarchar
+AS 'babelfishpg_tsql', 'tsql_openxml_get_colpattern'
+LANGUAGE C STRICT;
+
 CREATE OR REPLACE FUNCTION sys.sp_datatype_info_helper(
     IN odbcVer smallint,
     IN is_100 bool,

contrib/babelfishpg_tsql/sql/sys_procedures.sql

@@ -333,3 +333,20 @@ GRANT EXECUTE on PROCEDURE sys.sp_enum_oledb_providers() TO PUBLIC;
 CREATE OR REPLACE PROCEDURE sys.sp_reset_connection()
 AS 'babelfishpg_tsql', 'sp_reset_connection_internal' LANGUAGE C;
 GRANT EXECUTE ON PROCEDURE sys.sp_reset_connection() TO PUBLIC;
+
+CREATE OR REPLACE PROCEDURE sys.sp_xml_preparedocument( INOUT "@hdoc"  INTEGER,                 
+                                                              IN "@xmltext" sys.VARCHAR DEFAULT NULL,    
+                                                              IN "@xpath_namespaces" sys.VARCHAR DEFAULT NULL )
+AS 'babelfishpg_tsql', 'sp_xml_preparedocument'
+LANGUAGE C;
+
+GRANT EXECUTE ON PROCEDURE sys.sp_xml_preparedocument( INOUT "@hdoc" INTEGER,
+                                                             IN "@xmltext" sys.VARCHAR ,
+                                                             IN "@xpath_namespaces" sys.VARCHAR )
+TO PUBLIC;
+
+CREATE OR REPLACE PROCEDURE sys.sp_xml_removedocument( IN "@hdoc" INTEGER ) 
+AS 'babelfishpg_tsql', 'sp_xml_removedocument'
+LANGUAGE C;
+
+GRANT EXECUTE ON PROCEDURE sys.sp_xml_removedocument( IN INTEGER ) TO PUBLIC;

contrib/babelfishpg_tsql/sql/upgrades/babelfishpg_tsql--5.2.0--5.3.0.sql

@@ -1009,6 +1009,36 @@ STABLE;
 -- Please have this be one of the last statements executed in this upgrade script.
 DROP PROCEDURE sys.babelfish_drop_deprecated_object(varchar, varchar, varchar);
 
+CREATE OR REPLACE PROCEDURE sys.sp_xml_preparedocument(
+    INOUT "@hdoc"  INTEGER,                 
+    IN "@xmltext" sys.VARCHAR DEFAULT NULL,    
+    IN "@xpath_namespaces" sys.VARCHAR DEFAULT NULL 
+) 
+AS 'babelfishpg_tsql', 'sp_xml_preparedocument'
+LANGUAGE C;
+GRANT EXECUTE ON PROCEDURE sys.sp_xml_preparedocument(
+	INOUT INTEGER, IN sys.varchar, IN sys.varchar
+) TO PUBLIC;
+
+CREATE OR REPLACE PROCEDURE sys.sp_xml_removedocument(
+    IN "@hdoc" INTEGER
+) 
+AS 'babelfishpg_tsql', 'sp_xml_removedocument'
+LANGUAGE C;
+GRANT EXECUTE ON PROCEDURE sys.sp_xml_removedocument(
+	IN INTEGER
+) TO PUBLIC;
+
+CREATE OR REPLACE FUNCTION sys.tsql_openxml_get_xmldoc(int)
+RETURNS xml
+AS 'babelfishpg_tsql', 'tsql_openxml_get_xmldoc'
+LANGUAGE C STRICT;
+
+CREATE OR REPLACE FUNCTION sys.tsql_openxml_get_colpattern(text,int)
+RETURNS sys.nvarchar
+AS 'babelfishpg_tsql', 'tsql_openxml_get_colpattern'
+LANGUAGE C STRICT;
+
 -- After upgrade, always run analyze for all babelfish catalogs.
 CALL sys.analyze_babelfish_catalogs();
 -- Reset search_path to not affect any subsequent scripts

contrib/babelfishpg_tsql/src/backend_parser/gram-tsql-decl.y

@@ -41,6 +41,9 @@
 %type <list> openjson_col_defs
 %type <str> optional_path
 %type <boolean> optional_asJson
+%type <node> openxml_expr
+%type <list> openxml_column_list
+%type <node> openxml_column_el
 
 %type <node> tsql_opt_arg_dflt
 %type <node> tsql_opt_null_keyword
@@ -117,7 +120,7 @@
 	TSQL_NOCHECK TSQL_NOLOCK TSQL_READUNCOMMITTED TSQL_UPDLOCK TSQL_REPEATABLEREAD
 	TSQL_READCOMMITTED TSQL_TABLOCK TSQL_TABLOCKX TSQL_PAGLOCK TSQL_ROWLOCK
 	TSQL_TOP TSQL_PERCENT
-	TSQL_AUTO TSQL_EXPLICIT TSQL_RAW TSQL_PATH TSQL_FOR TSQL_BASE64 TSQL_ROOT TSQL_READPAST TSQL_XLOCK TSQL_NOEXPAND OPENJSON JSON_MODIFY
+	TSQL_AUTO TSQL_EXPLICIT TSQL_RAW TSQL_PATH TSQL_FOR TSQL_BASE64 TSQL_ROOT TSQL_READPAST TSQL_XLOCK TSQL_NOEXPAND OPENJSON OPENXML JSON_MODIFY
 	TSQL_JSON TSQL_INCLUDE_NULL_VALUES TSQL_WITHOUT_ARRAY_WRAPPER
 	TSQL_MEMBER TSQL_SERVER
 	TSQL_WINDOWS TSQL_CERTIFICATE TSQL_DEFAULT_DATABASE TSQL_DEFAULT_LANGUAGE TSQL_HASHED
@@ -134,6 +137,6 @@
  * otherwise the parser cannot tell between 'WITH' and 'WITH (' and thus
  * lead to a shift/reduce conflict.
  */
-%token	WITH_paren TSQL_HINT_START_BRACKET UPDATE_paren
+%token	WITH_paren TSQL_HINT_START_BRACKET UPDATE_paren WITH_table
 
 %left TSQL_CROSS TSQL_OUTER TSQL_UNPIVOT

contrib/babelfishpg_tsql/src/backend_parser/gram-tsql-rule.y

@@ -1779,13 +1779,106 @@ table_ref:	relation_expr tsql_table_hint_expr
 					 */
 					$$ = (Node *) $1;
 				}
+			| TSQL_APPLY openxml_expr
+				{
+					/*
+					 * This case handles openxml cross/outer apply
+					 */
+					$$ = (Node *) $2;
+				}
+			| openxml_expr
+				{
+					/*
+					 * Standard openxml case
+					 */
+					$$ = (Node *) $1;
+				}
 			| table_ref TSQL_UNPIVOT tsql_unpivot_clause alias_clause
 				{
 					List *unpivot_info = list_make3($1, (List *)$3, $4);
                     $$ = tsql_unpivot_transformation(unpivot_info, @1);
 				}
 		;
 
+openxml_expr: OPENXML '(' a_expr ',' a_expr ')' WITH_table TABLE qualified_name opt_alias_clause
+				{
+					RangeTableFunc *n = makeNode(RangeTableFunc);
+					n->docexpr = NULL;
+					n->rowexpr = $5;
+					n->location = @1;
+					/* Default flag is 0 when not specified */
+					n->namespaces = list_make2((Node *)$3, makeIntConst(0, @1));
+					n->columns = list_make1($9);
+					n->alias = $10;
+					$$ = (Node *) n;
+				}
+			| OPENXML '(' a_expr ',' a_expr ',' a_expr ')' WITH_table TABLE qualified_name opt_alias_clause
+				{
+					RangeTableFunc *n = makeNode(RangeTableFunc);
+					n->docexpr = NULL;
+					n->rowexpr = $5;
+					n->location = @1;
+					n->namespaces = list_make2((Node *)$3, (Node *)$7);
+					n->columns = list_make1($11);
+					n->alias = $12;
+					$$ = (Node *) n;
+				}
+			| OPENXML '(' a_expr ',' a_expr ')' WITH_paren '(' openxml_column_list ')' opt_alias_clause
+				{
+					RangeTableFunc *n = makeNode(RangeTableFunc);
+					n->docexpr = NULL;
+					n->rowexpr = $5;
+					n->location = @1;
+					/* Default flag is 0 when not specified */
+					n->columns = $9;
+					n->namespaces = list_make2((Node *)$3, makeIntConst(0, @1));
+					n->alias = $11;
+					$$ = (Node *) n;
+				}
+			| OPENXML '(' a_expr ',' a_expr ',' a_expr ')' WITH_paren '(' openxml_column_list ')' opt_alias_clause
+				{
+					RangeTableFunc *n = makeNode(RangeTableFunc);
+					n->docexpr = NULL;
+					n->rowexpr = $5;
+					n->columns = $11;
+					n->location = @1;
+					n->namespaces = list_make2((Node *)$3, (Node *)$7);
+					n->alias = $13;
+					$$ = (Node *) n;
+				}
+		;
+
+openxml_column_list: openxml_column_el                    { $$ = list_make1($1); }
+            | openxml_column_list ',' openxml_column_el    { $$ = lappend($1, $3); }
+        ;
+
+openxml_column_el:
+			ColId Typename
+				{
+					RangeTableFuncCol *fc = makeNode(RangeTableFuncCol);
+
+					fc->colname = $1;
+					fc->typeName = $2;
+					fc->colexpr = NULL;
+					fc->coldefexpr = NULL;
+					fc->location = @1;
+
+					$$ = (Node *) fc;
+				}	
+			| ColId Typename Sconst
+				{
+					RangeTableFuncCol *fc = makeNode(RangeTableFuncCol);
+
+					fc->colname = $1;
+					fc->typeName = $2;
+					fc->colexpr = (Node *) makeStringConst($3, @1);
+					fc->coldefexpr = NULL;
+					fc->location = @1;
+
+					$$ = (Node *) fc;
+				}	
+		;
+
 openjson_expr: OPENJSON '(' a_expr  ')' opt_alias_clause
 				{
 					RangeFunction *n = makeNode(RangeFunction);

contrib/babelfishpg_tsql/src/backend_parser/kwlist.h

@@ -323,6 +323,7 @@ PG_KEYWORD("old_password", TSQL_OLD_PASSWORD, UNRESERVED_KEYWORD)
 PG_KEYWORD("on", ON, RESERVED_KEYWORD)
 PG_KEYWORD("only", ONLY, RESERVED_KEYWORD)
 PG_KEYWORD("openjson", OPENJSON, COL_NAME_KEYWORD)
+PG_KEYWORD("openxml", OPENXML, COL_NAME_KEYWORD)
 PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD)
 PG_KEYWORD("option", OPTION, RESERVED_KEYWORD)
 PG_KEYWORD("options", OPTIONS, UNRESERVED_KEYWORD)

contrib/babelfishpg_tsql/src/backend_parser/parser.c

@@ -266,6 +266,8 @@ pgtsql_base_yylex(YYSTYPE *lvalp, YYLTYPE * llocp, core_yyscan_t yyscanner)
 				case '(':
 					cur_token = WITH_paren;
 					break;
+				case TABLE:
+					cur_token = WITH_table;
 			}
 			break;
 		case '(':

contrib/babelfishpg_tsql/src/extendedproperty.c

@@ -71,7 +71,6 @@ static void init_scan_key(ScanKeyData *scanKey,
 static void sp_execextended_property(PG_FUNCTION_ARGS, ExtendedPropertyProc proc);
 static bool get_extended_property_from_tuple(Relation relation, HeapTuple tuple,
 											 Datum *values, bool *nulls, int len);
-static char* get_value_by_name_from_array(ArrayType *array, const char *name);
 
 static void
 init_scan_key(ScanKeyData *scanKey,
@@ -923,7 +922,7 @@ fn_listextendedproperty(PG_FUNCTION_ARGS)
 extern const char *ATTOPTION_BBF_ORIGINAL_TABLE_NAME;
 extern const char *ATTOPTION_BBF_ORIGINAL_NAME;
 
-static char*
+char*
 get_value_by_name_from_array(ArrayType *array, const char *name)
 {
 	int			i;

contrib/babelfishpg_tsql/src/extendedproperty.h

@@ -16,6 +16,7 @@ typedef enum ExtendedPropertyType
 } ExtendedPropertyType;
 
 extern const char *const ExtendedPropertyTypeNames[];
+extern char *get_value_by_name_from_array(ArrayType *array, const char *name);
 
 extern void delete_extended_property(int16 db_id,
                                      const char *type,