8000 fix(mysql): type an aggregate under group by by its argument by colinhacks · Pull Request #120 · colinhacks/reylo · GitHub
[go: up one dir, main page]

Skip to content
8000
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading

Uh oh!

There was an error while loading. Please reload this page.

Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion engine/mysql/src/mysql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,11 @@ fn prove(q: &Query, raw: Option<&MysqlDescribe>, views: &HashSet<String>, host:
None => pending = true,
}
}
// An aggregate under a plain GROUP BY: the argument's flag over the rows it aggregates.
match grouped_flags(q, host) {
Some(found) => not_null.extend(found),
None => pending = true,
}
// A window aggregate over a frame that holds the current row: the argument's flag there.
match window_flags(q, count, host) {
Some(found) => not_null.extend(found),
Expand Down Expand Up @@ -687,7 +692,28 @@ fn window_flags(q: &Query, count: Option<usize>, host: &Host) -> Lazy<Vec<usize>
if let Ok(d) = host.describe(&probe.to_string())? {
if Some(d.columns.len()) == count {
let tested = crate::mysql_nulls::tested(&probe);
found.extend(follows.into_iter().filter(|&i| d.columns.get(i).is_some_and(|c| c.flags & 1 != 0) || tested.contains(&i)));
// An argument that is an aggregate of a plain GROUP BY (`sum(max(x)) over ()`)
// is as NULL as that aggregate's own argument.
let grouped = grouped_flags(&probe, host)?;
found.extend(follows.into_iter().filter(|&i| d.columns.get(i).is_some_and(|c| c.flags & 1 != 0) || tested.contains(&i) || grouped.contains(&i)));
}
}
}
Some(found)
}

/// The positions of the columns that are an aggregate under a plain GROUP BY
/// ([`crate::mysql_nulls::grouped_aggregate`]) and are never NULL: a JSON aggregate, and one that
/// follows its argument where the statement that selects the arguments from the same FROM and
/// WHERE flags that argument NOT NULL or the WHERE proves it. The server flags every such call
/// nullable but the bit functions and `count`.
fn grouped_flags(q: &Query, host: &Host) -> Lazy<Vec<usize>> {
let Some((probe, follows, mut found)) = crate::mysql_nulls::grouped_probe(q) else { return Some(Vec::new()) };
if !follows.is_empty() {
if let Ok(d) = host.describe(&probe.to_string())? {
if d.columns.len() == follows.len() {
let tested = crate::mysql_nulls::tested(&probe);
found.extend(follows.into_iter().enumerate().filter(|&(j, _)| d.columns[j].flags & 1 != 0 || tested.contains(&j)).map(|(_, i)| i));
}
}
}
Expand Down
96 changes: 96 additions & 0 deletions engine/mysql/src/mysql_nulls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,78 @@ pub(crate) fn window_probe(q: &Query) -> Option<(Query, Vec<usize>, Vec<usize>)>
Some((probe, follows, always))
}

/// What a select-list item of a SELECT with a plain GROUP BY says of its column when it is an
/// aggregate of its own, not a window call: every group holds a row, so `sum`, `avg`, `min` and
/// `max` (DISTINCT or not) follow their argument over the group's rows, and `json_arrayagg` and
/// `json_objectagg` are never NULL. `None` for anything else.
pub(crate) fn grouped_aggregate(e: &Expr) -> Option<Windowed<'_>> {
let Expr::Function(f) = strip(e) else { return None };
if f.over.is_some() || f.filter.is_some() || !f.within_group.is_empty() || f.null_treatment.is_some() {
return None;
}
let ast::FunctionArguments::List(list) = &f.args else { return None };
if !list.clauses.is_empty() {
return None;
}
let args: Vec<&Expr> = list
.args
.iter()
.map(|a| match a {
ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(x)) => Some(x),
_ => None,
})
.collect::<Option<_>>()?;
let name = crate::mysql::builtin(f);
match (name.as_str(), args.as_slice()) {
(n, [arg]) if WINDOW_FOLLOWING.contains(&n) => Some(Windowed::Follows(arg)),
("json_arrayagg", [_]) | ("json_objectagg", [_, _]) if list.duplicate_treatment.is_none() => Some(Windowed::Always),
_ => None,
}
}

/// For a SELECT with a plain GROUP BY: the statement that selects, from the same FROM and WHERE
/// with no GROUP BY, the argument of each select-list item [`grouped_aggregate`] finds following
/// its argument, one column each, beside the positions of those items and of the ones never NULL.
/// Prepared, the copy's flag for a column is the argument's on every row a group aggregates, read
/// from the rows before they are grouped as the aggregate reads them. `None` for anything but a
/// plain SELECT with a plain GROUP BY and such an item, and where a wildcard makes the positions
/// unreadable. A ROLLUP is left to the server's flags, as Postgres leaves a grouping set, though
/// measured on 8.4 it adds no row over no rows and its summary rows aggregate the rows themselves.
pub(crate) fn grouped_probe(q: &Query) -> Option<(Query, Vec<usize>, Vec<usize>)> {
let SetExpr::Select(select) = q.body.as_ref() else { return None };
if !grouped(&select.group_by) || select.projection.iter().any(|i| matches!(i, SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(..))) {
return None;
}
let (mut follows, mut always, mut args) = (Vec::new(), Vec::new(), Vec::new());
for (i, item) in select.projection.iter().enumerate() {
let (SelectItem::UnnamedExpr(e) | SelectItem::ExprWithAlias { expr: e, .. }) = item else { continue };
match grouped_aggregate(e) {
Some(Windowed::Follows(arg)) => {
follows.push(i);
args.push(SelectItem::UnnamedExpr(arg.clone()));
}
Some(Windowed::Always) => always.push(i),
None => {}
}
}
if follows.is_empty() && always.is_empty() {
return None;
}
let mut probe = q.clone();
probe.order_by = None;
probe.limit_clause = None;
probe.locks.clear();
if let SetExpr::Select(select) = probe.body.as_mut() {
select.projection = args;
select.distinct = None;
select.group_by = GroupByExpr::Expressions(Vec::new(), Vec::new());
select.having = None;
select.named_window.clear();
select.qualify = None;
}
Some((probe, follows, always))
}

/// Whether a SELECT has a plain GROUP BY, where every group has a row.
pub(crate) fn grouped(group_by: &GroupByExpr) -> bool {
matches!(group_by, GroupByExpr::Expressions(exprs, _) if !exprs.is_empty()) && !rolls_up(group_by)
Expand Down Expand Up @@ -603,4 +675,28 @@ mod tests {
assert_eq!(probe("select *, sum(a) over () from t"), None, "a wildcard");
assert_eq!(probe("select a, sum(a) over () from t group by a with rollup"), None, "a ROLLUP");
}

#[test]
fn an_aggregate_under_a_plain_group_by_follows_its_argument() {
let probe = |sql: &str| grouped_probe(&query(sql)).map(|(q, follows, always)| (q.to_string(), follows, always));
// The arguments, read from the rows before they are grouped, and the JSON aggregates.
assert_eq!(
probe("select distinct user_id, sum(a) as s, max(distinct b), count(*), json_arrayagg(b) as j, json_objectagg(a, b) from t where a > 0 group by user_id having count(*) > 1 order by s limit 2"),
Some(("SELECT a, b FROM t WHERE a > 0".into(), vec![1, 2], vec![4, 5]))
);
assert_eq!(probe("with w as (select a from t) select a, min(a) from w group by a"), Some(("WITH w AS (SELECT a FROM t) SELECT a FROM w".into(), vec![1], vec![])));
// No GROUP BY, a ROLLUP, a window call, a wildcard, an expression over an aggregate and
// the aggregates that are NULL otherwise prove nothing.
for sql in [
"select sum(a) from t",
"select user_id, sum(a) from t group by user_id with rollup",
"select user_id, sum(a) over () from t group by user_id, a",
"select *, sum(a) from t group by user_id",
"select user_id, sum(a) + 1 from t group by user_id",
"select user_id, var_samp(a), stddev_samp(a), group_concat(a), `sum`(a), app.sum(a) from t group by user_id",
"select user_id, a from t union all select user_id, max(a) from t group by user_id",
] {
assert_eq!(probe(sql), None, "{sql}");
}
}
}
2 changes: 1 addition & 1 deletion oracle/cases/mysql_booleans/expected.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ aggregates.sql: one () -> { most: int8|int4?, total: numeric?, some: bool }
buffered_and_sorted.sql: many () -> { done: bool, eq: bool? }
columns.sql: many () -> { done: bool, maybe: bool?, unsigned_flag: bool?, small: int2?, four: int2?, bit_flag: bool?, eight: bytea? }
comparisons.sql: many () -> { eq: bool?, ne: bool?, lt: bool?, nse: bool, isnull: bool, nottrue: bool, inl: bool?, btw: bool?, lk: bool?, re: bool?, nota: bool?, andab: bool?, orab: bool?, xorab: bool?, t: bool, f: bool }
expressions_over_flags.sql: many () -> { zero: int8|int4, most: int2?, plus: int8|int4?, either: int2? }
expressions_over_flags.sql: many () -> { zero: int8|int4, most: int2, plus: int8|int4?, either: int2? }
not_truth_values.sql: many () -> { one: int8|int4, five: int8|int4, cs: int8|int4, mixed: int8|int4?, plus: int8|int4? }
passed_through.sql: many () -> { co: bool, ifn: bool?, iff: bool?, cs: bool?, ni: bool?, sub: bool?, ex: bool }
placeholders.sql: many (n: int4, m: int4, s: text, p: int4) -> { eq: bool?, listed: bool?, lk: bool?, co: bool? }
Expand Down
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/by_expression.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select date_format(p.published_at, '%Y') as y, count(*) as n, max(p.id) as top from posts p group by date_format(p.published_at, '%Y')
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/derived.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select d.user_id, d.top from (select user_id, max(id) as top from posts group by user_id) d
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/dialect
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
mysql
20 changes: 20 additions & 0 deletions oracle/cases/mysql_grouped_aggregates/expected.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
by_expression.sql: many () -> { y: text?, n: int8, top: int4|int8 }
derived.sql: many () -> { user_id: int4, top: int4|int8 }
expression.sql: many () -> { user_id: int4, next: numeric?, note: text }
grouped.sql: many () -> { user_id: int4, total: numeric, mean: numeric, first_title: text, last_at: timestamp?, score: float8? }
grouped_distinct.sql: many () -> { user_id: int4, total: numeric, top: text }
grouped_window.sql: many () -> { user_id: int4, top: int4|int8, total: numeric }
having.sql: many () -> { user_id: int4, top: text }
json.sql: many () -> { user_id: int4, by_title: json, notes: [text?] }
left_joined.sql: many () -> { id: int4, top: text? }
left_joined_reduced.sql: many () -> { id: int4, top: text }
param.sql: many (1: int4) -> { user_id: int4, top: int4|int8 }
rollup.sql: many () -> { user_id: int4|int8?, total: numeric?, top: text? }
scalar_subquery.sql: many () -> { id: int4, top: int4|int8? }
select_distinct.sql: many () -> { user_id: int4, top: int4|int8 }
ungrouped.sql: one () -> { total: numeric?, top: text? }
ungrouped_empty.sql: one () -> { top: int4|int8? }
union_arm.sql: many () -> { user_id: int8, top: int8? }
variance.sql: many () -> { user_id: int4, v: float8?, s: float8?, g: text?, b: int8 }
where_tested.sql: many () -> { user_id: int4, last_note: text }
with_query.sql: many () -> { user_id: int4, top: int4|int8 }
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/expression.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, sum(p.id) + 1 as next, coalesce(max(p.note), '') as note from posts p group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/grouped.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, sum(p.id) as total, avg(p.id) as mean, min(p.title) as first_title, max(p.published_at) as last_at, sum(p.score) as score from posts p group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/grouped_distinct.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, sum(distinct p.id) as total, max(distinct p.title) as top from posts p group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/grouped_window.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, max(p.id) as top, sum(max(p.id)) over () as total from posts p group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/having.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, max(p.title) as top from posts p group by p.user_id having count(*) > 1
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/json.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, json_objectagg(p.title, p.note) as by_title, json_arrayagg(p.note) as notes from posts p group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/left_joined.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select u.id, max(p.title) as top from users u left join posts p on p.user_id = u.id group by u.id
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select u.id, max(p.title) as top from users u left join posts p on p.user_id = u.id where p.id > 0 group by u.id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/param.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, max(p.id) as top from posts p where p.user_id = ? group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/rollup.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, sum(p.id) as total, max(p.title) as top from posts p group by p.user_id with rollup
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/scalar_subquery.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select u.id, (select max(p.id) from posts p where p.user_id = u.id group by p.user_id) as top from users u
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/select_distinct.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select distinct p.user_id, max(p.id) as top from posts p group by p.user_id
10 changes: 10 additions & 0 deletions oracle/cases/mysql_grouped_aggregates/setup.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
-- Under a plain GROUP BY every group holds a row, so an aggregate that is NULL only over no rows or
-- over nothing but NULLs (`sum`, `avg`, `min`, `max`) is as NULL as its argument, and
-- `json_arrayagg` and `json_objectagg` are never NULL; the server flags them nullable all the
-- same. Without a GROUP BY an empty table gives one row of NULLs, and the variance family is NULL
-- over one row (`var_samp`), so neither is claimed. Users without posts, posts without a date or a
-- note, and every edge value are here, so a wrong non-null meets a NULL.
create table users (id int primary key auto_increment, name varchar(40) not null);
create table posts (id int primary key auto_increment, user_id int not null, title varchar(80) not null, published_at datetime, score double, note varchar(20), key (user_id));
insert into users (name) values ('Ada'), ('Grace'), ('Linus');
insert into posts (user_id, title, published_at, score, note) values (1, 'Hello', '2026-01-01 00:00:00', 1.5, 'n'), (1, 'Draft', null, null, null), (2, 'Notes', '2026-02-03 00:00:00', 2, null);
Empty file.
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/ungrouped.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select sum(p.id) as total, max(p.title) as top from posts p
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/ungrouped_empty.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select max(p.id) as top from posts p where p.id > 100
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/union_arm.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select user_id, max(id) as top from posts group by user_id union all select 0, 1
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/variance.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, var_samp(p.id) as v, stddev_pop(p.id) as s, group_concat(p.title) as g, bit_or(p.id) as b from posts p group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/where_tested.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select p.user_id, max(p.note) as last_note from posts p where p.note is not null group by p.user_id
1 change: 1 addition & 0 deletions oracle/cases/mysql_grouped_aggregates/with_query.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
with t as (select user_id, id from posts) select user_id, max(id) as top from t group by user_id
2 changes: 1 addition & 1 deletion oracle/cases/mysql_server_flags/expected.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
aggregates.sql: one () -> { a: int8, b: int8, c: int8, d: numeric?, e: numeric?, f: int4|int8?, g: text?, h: date?, i: text?, j: int8, k: int8, l: int8, m: float8?, o: float8?, p: [int4]?, q: json?, r: numeric?, s: int8|int4?, u: int4|int8?, v: int8, w: numeric, x: bool, z: timestamp?, aa: numeric?, ab: float8?, ac: json?, ad: text?, ae: text?, af: bytea?, ag: int2?, ah: int2?, ai: time?, aj: float8?, ak: float8? }
aggregates_grouped.sql: many () -> { n: int4, a: int8, b: numeric?, c: text?, d: int4|int8?, e: text?, f: text, g: int8, h: numeric?, i: int8, j: date?, k: [int2] }
aggregates_grouped.sql: many () -> { n: int4, a: int8, b: numeric, c: text, d: int4|int8?, e: text?, f: text, g: int8, h: numeric, i: int8, j: date, k: [int2] }
aggregates_no_rows.sql: one () -> { a: int8, b: numeric?, c: int4|int8?, d: int8, e: int8, f: int8, g: int8, h: numeric?, i: float8?, j: text?, k: [int4]?, l: numeric?, m: int4|int8?, o: int4|int8?, p: bool }
aggregates_rollup.sql: many () -> { n: int4|int8?, small: int2?, a: int8, b: numeric?, c: int8?, d: text? }
bit_and_enum.sql: many () -> { a: bytea, b: int8|int4, c: text?, d: text, e: float8, f: text, g: float8, h: bool, i: bool, j: int2, k: int8|int4, l: bool, m: int8|int4, o: int8, p: text?, q: text?, r: text?, s: text?, u: text?, v: int8, w: int8, x: int8 }
Expand Down
7 changes: 4 additions & 3 deletions oracle/cases/mysql_server_flags/setup.sql
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
-- Nullability is the server's NOT NULL flag, which the engine takes as it is: sound where it is
-- set, and nullable for string and date functions and most casts of NOT NULL inputs. Only what the
-- statement proves narrows it: a strict test in the WHERE (`engine/mysql/src/mysql_nulls.rs`)
-- proves the column it tests and the row of an outer join, a GROUP BY's JSON_ARRAYAGG is never
-- NULL, and an aggregate over a window frame that holds the current row is as NULL as its argument
-- (`window_flags` in `engine/mysql/src/mysql.rs`). Every
-- proves the column it tests and the row of an outer join, an aggregate under a plain GROUP BY and
-- one over a window frame that holds the current row are as NULL as their argument, and their
-- JSON_ARRAYAGG and JSON_OBJECTAGG are never NULL (`grouped_flags` and `window_flags` in
-- `engine/mysql/src/mysql.rs`). Every
-- column here is NOT NULL but those named `maybe_`, and the edge state holds zero dates, the
-- bounds of every integer and an empty string.
create table t (
Expand Down
2 changes: 1 addition & 1 deletion oracle/cases/mysql_window_aggregates/expected.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ chained_windows.sql: many () -> { total: numeric, ahead: numeric? }
copied_window.sql: many () -> { around: numeric, before_row: numeric? }
count_window.sql: many () -> { n: int8 }
following_frame.sql: many () -> { ahead: numeric? }
grouped_sum.sql: many () -> { user_id: int4, total: numeric? }
grouped_sum.sql: many () -> { user_id: int4, total: numeric }
left_joined.sql: many () -> { name: text, total: numeric? }
monthly_running.sql: many () -> { month: text?, running: numeric }
named_window.sql: many () -> { total: numeric, last_title: text }
Expand Down
Binary file modified packages/mysql/engine.wasm
Binary file not shown.
19 changes: 17 additions & 2 deletions site/content/docs/mysql.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,23 @@ from users

</Inferred>

**Aggregates**

Without a `group by`, an aggregate other than `count`, `bit_and`, `bit_or` and `bit_xor` returns `NULL` over no rows. With one, every group has rows: `json_arrayagg` and `json_objectagg` are non-null, and `sum`, `avg`, `min` and `max` are non-null when their argument is.

<Inferred>

```sql file="my_null_aggregates.sql" dialect="mysql"
select user_id,
json_arrayagg(id) as order_ids,
max(ships_on) as last_ships_on,
sum(total) as spent
from orders
group by user_id
```

</Inferred>

**Subqueries**

A scalar subquery that aggregates without a `group by` returns one row, so its `count` is non-null, whether it reads a table, a subquery in `from` or a `with` query. A column read from a `with` query or a subquery in `from` is as nullable as it is in that query.
Expand Down Expand Up @@ -419,8 +436,6 @@ group by role with rollup

</Rules>

`json_arrayagg` returns `NULL` over no rows, but is non-null with a `group by`, since every group has rows.

To tell Reylo a value is never null, give it a default with `coalesce` or `ifnull`, or filter out the nulls with `where x is not null`.

## Cardinality
Expand Down
Loading
Loading
0