Skip to content

Commit 3699802

Browse files
ArjunNarendraanushakolan
authored and
naxing123
committed
Add groupby/aggregation support for PostgreSQL in GraphQL (#3741)
## Why make this change? Closes #2850. This PR adds support for group by and aggregation in PostgreSQL in GraphQL queries. ## What is this change? Groupby and aggregation types/enums were added to the GraphQL schema in PostgreSQL, but they essentially were orphaned because the user couldn't utilize them in their GraphQL query. This PR hooks up the groupby/aggregation schema types/enums into a GraphQL input query. ## How was this tested? - [X] Integration Tests ## Sample Request(s) ``` query { stocks_prices { groupBy(fields: [categoryid, pieceid]) { fields { categoryid pieceid } aggregations { total: sum(field: price) count: count(field: pieceid, having: { gt: 5 }) } } } } ``` Previously, a query such as the one above would fail because the `groupBy` field would not have been attached to the `stocks_prices` object type. This PR adds the `groupBy` field to the relevant list query entity types. The `groupBy` field serves as a window to the rest of the groupBy/aggregation query. --------- Co-authored-by: Anusha Kolan <anushakolan10@gmail.com> (cherry picked from commit 7ad4f49)
1 parent 7e0eebf commit 3699802

13 files changed

Lines changed: 602 additions & 203 deletions

‎config-generators/postgresql-commands.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,3 +176,4 @@ add dbo_DimAccount --config "dab-config.PostgreSql.json" --source "dimaccount" -
176176
update dbo_DimAccount --config "dab-config.PostgreSql.json" --map "parentaccountkey:ParentAccountKey,accountkey:AccountKey"
177177
update dbo_DimAccount --config "dab-config.PostgreSql.json" --relationship parent_account --target.entity dbo_DimAccount --cardinality one --relationship.fields "parentaccountkey:accountkey"
178178
update dbo_DimAccount --config "dab-config.PostgreSql.json" --relationship child_accounts --target.entity dbo_DimAccount --cardinality many --relationship.fields "accountkey:parentaccountkey"
179+
add DateOnlyTable --config "dab-config.PostgreSql.json" --source "date_only_table" --permissions "anonymous:*" --rest true --graphql true --source.key-fields "event_date"

‎src/Core/Resolvers/BaseSqlQueryBuilder.cs‎

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,92 @@ protected virtual string Build(AggregationColumn column, bool useAlias = false)
193193
return $"{column.Type.ToString()}({columnName}) {appendAlias}";
194194
}
195195

196+
/// <summary>
197+
/// Build the Group By Clause needed to append to the main query
198+
/// </summary>
199+
/// <param name="structure">Sql query structure to build query on</param>
200+
/// <returns>SQL query with group-by clause</returns>
201+
protected virtual string BuildGroupBy(SqlQueryStructure structure)
202+
{
203+
// Add GROUP BY clause if there are any group by columns
204+
if (structure.GroupByMetadata.Fields.Any())
205+
{
206+
return $" GROUP BY {string.Join(", ", structure.GroupByMetadata.Fields.Values.Select(c => Build(c)))}";
207+
}
208+
209+
return string.Empty;
210+
}
211+
212+
/// <summary>
213+
/// Build the Having clause needed to append to the main query
214+
/// </summary>
215+
/// <param name="structure">Sql query structure to build query on</param>
216+
/// <returns>SQL query with having clause</returns>
217+
protected virtual string BuildHaving(SqlQueryStructure structure)
218+
{
219+
if (structure.GroupByMetadata.Aggregations.Count > 0)
220+
{
221+
List<Predicate>? havingPredicates = structure.GroupByMetadata.Aggregations
222+
.SelectMany(aggregation => aggregation.HavingPredicates ?? new List<Predicate>())
223+
.ToList();
224+
225+
if (havingPredicates.Any())
226+
{
227+
return $" HAVING {Build(havingPredicates)}";
228+
}
229+
}
230+
231+
return string.Empty;
232+
}
233+
234+
/// <summary>
235+
/// Build the aggregation columns needed to append to the main query
236+
/// </summary>
237+
/// <param name="structure">Sql query structure to build query on</param>
238+
/// <returns>SQL query with aggregation columns</returns>
239+
protected virtual string BuildAggregationColumns(SqlQueryStructure structure)
240+
{
241+
string aggregations = string.Empty;
242+
if (structure.GroupByMetadata.Aggregations.Count > 0)
243+
{
244+
if (structure.Columns.Any())
245+
{
246+
aggregations = $",{BuildAggregationColumns(structure.GroupByMetadata)}";
247+
}
248+
else
249+
{
250+
aggregations = $"{BuildAggregationColumns(structure.GroupByMetadata)}";
251+
}
252+
}
253+
254+
return aggregations;
255+
}
256+
257+
/// <summary>
258+
/// Build the aggregation columns needed to append to the main query
259+
/// </summary>
260+
/// <param name="metadata">GroupByMetadata</param>
261+
/// <returns>SQL query with aggregation columns</returns>
262+
protected virtual string BuildAggregationColumns(GroupByMetadata metadata)
263+
{
264+
return string.Join(", ", metadata.Aggregations.Select(aggregation => Build(aggregation.Column, useAlias: true)));
265+
}
266+
267+
/// <summary>
268+
/// Build the Order By clause needed to append to the main query
269+
/// </summary>
270+
/// <param name="structure">Sql query structure to build query on</param>
271+
/// <returns>SQL query with order-by clause</returns>
272+
protected virtual string BuildOrderBy(SqlQueryStructure structure)
273+
{
274+
if (structure.OrderByColumns.Any())
275+
{
276+
return $" ORDER BY {Build(structure.OrderByColumns)}";
277+
}
278+
279+
return string.Empty;
280+
}
281+
196282
/// <summary>
197283
/// Build orderby column as
198284
/// {SourceAlias}.{ColumnName} {direction}
@@ -447,7 +533,7 @@ public virtual string BuildForeignKeyInfoQuery(int numberOfParameters)
447533
// constraint columns - one inner join for the columns from the 'Referencing table'
448534
// and the other join for the columns from the 'Referenced Table'.
449535
string foreignKeyQuery = $@"
450-
SELECT
536+
SELECT
451537
ReferentialConstraints.CONSTRAINT_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition))},
452538
ReferencingColumnUsage.TABLE_SCHEMA
453539
{QuoteIdentifier($"Referencing{nameof(DatabaseObject.SchemaName)}")},
@@ -457,9 +543,9 @@ public virtual string BuildForeignKeyInfoQuery(int numberOfParameters)
457543
{QuoteIdentifier($"Referenced{nameof(DatabaseObject.SchemaName)}")},
458544
ReferencedColumnUsage.TABLE_NAME {QuoteIdentifier($"Referenced{nameof(SourceDefinition)}")},
459545
ReferencedColumnUsage.COLUMN_NAME {QuoteIdentifier(nameof(ForeignKeyDefinition.ReferencedColumns))}
460-
FROM
546+
FROM
461547
INFORMATION_SCHEMA.REFERENTIAL_CONSTRAINTS ReferentialConstraints
462-
INNER JOIN
548+
INNER JOIN
463549
INFORMATION_SCHEMA.KEY_COLUMN_USAGE ReferencingColumnUsage
464550
ON ReferentialConstraints.CONSTRAINT_CATALOG = ReferencingColumnUsage.CONSTRAINT_CATALOG
465551
AND ReferentialConstraints.CONSTRAINT_SCHEMA = ReferencingColumnUsage.CONSTRAINT_SCHEMA

‎src/Core/Resolvers/BaseTSqlQueryBuilder.cs‎

Lines changed: 0 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
using Azure.DataApiBuilder.Config.ObjectModel;
2-
using Azure.DataApiBuilder.Core.Models;
32

43
namespace Azure.DataApiBuilder.Core.Resolvers
54
{
@@ -43,90 +42,5 @@ protected virtual string BuildPredicates(SqlQueryStructure structure)
4342
Build(structure.PaginationMetadata.PaginationPredicate));
4443
}
4544

46-
/// <summary>
47-
/// Build the Group By Clause needed to append to the main query
48-
/// </summary>
49-
/// <param name="structure">Sql query structure to build query on</param>
50-
/// <returns>SQL query with group-by clause</returns>
51-
protected virtual string BuildGroupBy(SqlQueryStructure structure)
52-
{
53-
// Add GROUP BY clause if there are any group by columns
54-
if (structure.GroupByMetadata.Fields.Any())
55-
{
56-
return $" GROUP BY {string.Join(", ", structure.GroupByMetadata.Fields.Values.Select(c => Build(c)))}";
57-
}
58-
59-
return string.Empty;
60-
}
61-
62-
/// <summary>
63-
/// Build the Having clause needed to append to the main query
64-
/// </summary>
65-
/// <param name="structure">Sql query structure to build query on</param>
66-
/// <returns>SQL query with having clause</returns>
67-
protected virtual string BuildHaving(SqlQueryStructure structure)
68-
{
69-
if (structure.GroupByMetadata.Aggregations.Count > 0)
70-
{
71-
List<Predicate>? havingPredicates = structure.GroupByMetadata.Aggregations
72-
.SelectMany(aggregation => aggregation.HavingPredicates ?? new List<Predicate>())
73-
.ToList();
74-
75-
if (havingPredicates.Any())
76-
{
77-
return $" HAVING {Build(havingPredicates)}";
78-
}
79-
}
80-
81-
return string.Empty;
82-
}
83-
84-
/// <summary>
85-
/// Build the Order By clause needed to append to the main query
86-
/// </summary>
87-
/// <param name="structure">Sql query structure to build query on</param>
88-
/// <returns>SQL query with order-by clause</returns>
89-
protected virtual string BuildOrderBy(SqlQueryStructure structure)
90-
{
91-
if (structure.OrderByColumns.Any())
92-
{
93-
return $" ORDER BY {Build(structure.OrderByColumns)}";
94-
}
95-
96-
return string.Empty;
97-
}
98-
99-
/// <summary>
100-
/// Build the aggregation columns needed to append to the main query
101-
/// </summary>
102-
/// <param name="structure">Sql query structure to build query on</param>
103-
/// <returns>SQL query with aggregation columns</returns>
104-
protected virtual string BuildAggregationColumns(SqlQueryStructure structure)
105-
{
106-
string aggregations = string.Empty;
107-
if (structure.GroupByMetadata.Aggregations.Count > 0)
108-
{
109-
if (structure.Columns.Any())
110-
{
111-
aggregations = $",{BuildAggregationColumns(structure.GroupByMetadata)}";
112-
}
113-
else
114-
{
115-
aggregations = $"{BuildAggregationColumns(structure.GroupByMetadata)}";
116-
}
117-
}
118-
119-
return aggregations;
120-
}
121-
122-
/// <summary>
123-
/// Build the aggregation columns needed to append to the main query
124-
/// </summary>
125-
/// <param name="metadata">GroupByMetadata</param>
126-
/// <returns>SQL query with aggregation columns</returns>
127-
protected virtual string BuildAggregationColumns(GroupByMetadata metadata)
128-
{
129-
return string.Join(", ", metadata.Aggregations.Select(aggregation => Build(aggregation.Column, useAlias: true)));
130-
}
13145
}
13246
}

‎src/Core/Resolvers/PostgresQueryBuilder.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,14 @@ public string Build(SqlQueryStructure structure)
4141
Build(structure.Predicates),
4242
Build(structure.PaginationMetadata.PaginationPredicate));
4343

44-
string query = $"SELECT {MakeSelectColumns(structure)}"
44+
string aggregations = BuildAggregationColumns(structure);
45+
46+
string query = $"SELECT {MakeSelectColumns(structure)}{aggregations}"
4547
+ $" FROM {fromSql}"
4648
+ $" WHERE {predicates}"
47-
+ $" ORDER BY {Build(structure.OrderByColumns)}"
49+
+ BuildGroupBy(structure)
50+
+ BuildHaving(structure)
51+
+ BuildOrderBy(structure)
4852
+ $" LIMIT {structure.Limit()}";
4953

5054
string subqueryName = QuoteIdentifier($"subq{structure.Counter.Next()}");

‎src/Core/Resolvers/Sql Query Structures/SqlQueryStructure.cs‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,26 @@ private List<OrderByColumn> PrimaryKeyAsOrderByColumns()
378378
return _primaryKeyAsOrderByColumns;
379379
}
380380

381+
/// <summary>
382+
/// Exposes the groupBy fields of this structure as a list of OrderByColumn,
383+
/// giving groupBy queries a deterministic row order (in the absence of an explicit
384+
/// client-provided orderBy) since Postgres/MSSQL do not guarantee GROUP BY row order otherwise.
385+
/// </summary>
386+
private List<OrderByColumn> GroupByColumnsAsOrderByColumns()
387+
{
388+
List<OrderByColumn> groupByColumnsAsOrderByColumns = new();
389+
390+
foreach (Column column in GroupByMetadata.Fields.Values)
391+
{
392+
groupByColumnsAsOrderByColumns.Add(new OrderByColumn(tableSchema: column.TableSchema,
393+
tableName: column.TableName,
394+
columnName: column.ColumnName,
395+
tableAlias: column.TableAlias));
396+
}
397+
398+
return groupByColumnsAsOrderByColumns;
399+
}
400+
381401
/// <summary>
382402
/// Private constructor that is used for recursive query generation,
383403
/// for each subquery that's necessary to resolve a nested GraphQL
@@ -511,8 +531,9 @@ private SqlQueryStructure(
511531
}
512532
}
513533

514-
// primary key should only be added to order by for non groupby queries.
515-
OrderByColumns = isGroupByQuery ? [] : PrimaryKeyAsOrderByColumns();
534+
// groupBy queries default to ordering by the grouped columns (since Postgres/MSSQL
535+
// don't guarantee GROUP BY row order otherwise); non-groupBy queries default to the primary key.
536+
OrderByColumns = isGroupByQuery ? GroupByColumnsAsOrderByColumns() : PrimaryKeyAsOrderByColumns();
516537
if (IsListQuery && queryParams.ContainsKey(QueryBuilder.ORDER_BY_FIELD_NAME))
517538
{
518539
object? orderByObject = queryParams[QueryBuilder.ORDER_BY_FIELD_NAME];

‎src/Service.GraphQLBuilder/Queries/QueryBuilder.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public static class QueryBuilder
3636
{
3737
DatabaseType.MSSQL,
3838
DatabaseType.DWSQL,
39+
DatabaseType.PostgreSQL,
3940
};
4041

4142
/// <summary>

‎src/Service.Tests/DatabaseSchema-MsSql.sql‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
-- Copyright (c) Microsoft Corporation.
22
-- Licensed under the MIT License.
33

4+
SET QUOTED_IDENTIFIER ON;
5+
46
BEGIN TRANSACTION
57
DROP SECURITY POLICY IF EXISTS revenuesSecPolicy;
68
DROP FUNCTION IF EXISTS revenuesPredicate;
@@ -489,8 +491,8 @@ WITH cteN(Number) AS
489491
CROSS JOIN sys.all_columns AS s2
490492
)
491493
INSERT INTO bookmarks ([id], [bkname])
492-
SELECT
493-
[Number],
494+
SELECT
495+
[Number],
494496
'Test Item #' + format([Number], '00000')
495497
FROM cteN WHERE [Number] <= @UpperBound;
496498
SET IDENTITY_INSERT bookmarks OFF
@@ -503,8 +505,8 @@ WITH cteN(Number) AS
503505
CROSS JOIN sys.all_columns AS s2
504506
)
505507
INSERT INTO mappedbookmarks ([id], [bkname])
506-
SELECT
507-
[Number],
508+
SELECT
509+
[Number],
508510
'Test Item #' + format([Number], '00000')
509511
FROM cteN WHERE [Number] <= @UpperBound;
510512

@@ -822,7 +824,7 @@ CREATE TABLE date_only_table (
822824
event_timestamp datetime NOT NULL
823825
);
824826

825-
INSERT INTO date_only_table( event_date, event_time, event_timestamp)
826-
VALUES ('2023-01-01', '08:30:00', '2023-01-01 08:30:00'),
827-
('2023-02-15', '12:45:00', '2023-02-15 12:45:00'),
827+
INSERT INTO date_only_table( event_date, event_time, event_timestamp)
828+
VALUES ('2023-01-01', '08:30:00', '2023-01-01 08:30:00'),
829+
('2023-02-15', '12:45:00', '2023-02-15 12:45:00'),
828830
('2023-03-30', '17:15:00', '2023-03-30 17:15:00');

‎src/Service.Tests/DatabaseSchema-PostgreSql.sql‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ DROP TABLE IF EXISTS default_with_function_table;
4040
DROP TABLE IF EXISTS users;
4141
DROP TABLE IF EXISTS user_profiles;
4242
DROP TABLE IF EXISTS dimaccount;
43+
DROP TABLE IF EXISTS date_only_table;
4344
DROP FUNCTION IF EXISTS insertCompositeView;
4445

4546
DROP SCHEMA IF EXISTS foo;
@@ -349,14 +350,14 @@ INSERT INTO bookmarks (id, bkname)
349350
SELECT
350351
value,
351352
CONCAT('Test Item #' , value)
352-
FROM
353+
FROM
353354
GENERATE_SERIES(1, 10000, 1) as value;
354355

355356
INSERT INTO mappedbookmarks (id, bkname)
356357
SELECT
357358
value,
358359
CONCAT('Test Item #' , value)
359-
FROM
360+
FROM
360361
GENERATE_SERIES(1, 10000, 1) as value;
361362

362363
INSERT INTO GQLmappings(__column1, __column2, column3) VALUES (1, 'Incompatible GraphQL Name', 'Compatible GraphQL Name');
@@ -366,7 +367,7 @@ INSERT INTO GQLmappings(__column1, __column2, column3) VALUES (5, 'Filtered Reco
366367
INSERT INTO publishers(id, name) VALUES (1234, 'Big Company'), (2345, 'Small Town Publisher'), (2323, 'TBD Publishing One'), (2324, 'TBD Publishing Two Ltd'), (1940, 'Policy Publisher 01'), (1941, 'Policy Publisher 02'), (1156, 'The First Publisher');
367368
INSERT INTO clubs(id, name) VALUES (1111, 'Manchester United'), (1112, 'FC Barcelona'), (1113, 'Real Madrid');
368369
INSERT INTO players(id, name, current_club_id, new_club_id)
369-
VALUES
370+
VALUES
370371
(1, 'Cristiano Ronaldo', 1113, 1111),
371372
(2, 'Leonel Messi', 1112, 1113);
372373
INSERT INTO authors(id, name, birthdate) VALUES (123, 'Jelte', '2001-01-01'), (124, 'Aniruddh', '2002-02-02'), (125, 'Aniruddh', '2001-01-01'), (126, 'Aaron', '2001-01-01');
@@ -470,3 +471,14 @@ $$ LANGUAGE plpgsql;
470471

471472
CREATE TRIGGER insertCompositeViewTrigger INSTEAD OF INSERT ON books_publishers_view_composite_insertable
472473
FOR EACH ROW EXECUTE PROCEDURE insertCompositeView();
474+
475+
CREATE TABLE date_only_table (
476+
event_date date NOT NULL,
477+
event_time time NOT NULL,
478+
event_timestamp timestamp NOT NULL
479+
);
480+
481+
INSERT INTO date_only_table(event_date, event_time, event_timestamp)
482+
VALUES ('2023-01-01', '08:30:00', '2023-01-01 08:30:00'),
483+
('2023-02-15', '12:45:00', '2023-02-15 12:45:00'),
484+
('2023-03-30', '17:15:00', '2023-03-30 17:15:00');

‎src/Service.Tests/GraphQLBuilder/QueryBuilderTests.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -578,12 +578,12 @@ public void GenerateReturnType_ExcludesGroupByField_WhenAggregationDisabled()
578578
/// Tests that QueryBuilder.Build correctly adds or omits the groupBy field on the
579579
/// connection type based on whether the database type is in
580580
/// <see cref="QueryBuilder.AggregationEnabledDatabaseTypes"/>.
581-
/// MSSQL and DWSQL are supported; other types (e.g. PostgreSQL) are not.
581+
/// MSSQL, DWSQL, and PostgreSQL are supported; other types (e.g. MySQL) are not.
582582
/// </summary>
583583
[DataTestMethod]
584584
[DataRow(DatabaseType.MSSQL, true, DisplayName = "MSSQL: groupBy field present when aggregation enabled")]
585585
[DataRow(DatabaseType.DWSQL, true, DisplayName = "DWSQL: groupBy field present when aggregation enabled")]
586-
[DataRow(DatabaseType.PostgreSQL, false, DisplayName = "PostgreSQL: groupBy field absent (not in AggregationEnabledDatabaseTypes)")]
586+
[DataRow(DatabaseType.PostgreSQL, true, DisplayName = "PostgreSQL: groupBy field present when aggregation enabled")]
587587
[DataRow(DatabaseType.MySQL, false, DisplayName = "MySQL: groupBy field absent (not in AggregationEnabledDatabaseTypes)")]
588588
[TestCategory("Query Builder - Aggregation")]
589589
public void Build_WithAggregationEnabled_GroupByPresenceMatchesDatabaseSupport(

0 commit comments

Comments
 (0)