8000 fix(mysql): type a window aggregate over the current row as not null by colinhacks · Pull Request #115 · colinhacks/reylo · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions engine/mysql/src/mysql.rs
10BC8
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,11 @@ fn prove(q: &Query, raw: Option<&MysqlDescribe>, views: &HashSet<String>, host:
None => pending = true,
}
}
// A window aggregate over a frame that holds the current row: the argument's flag there.
match window_flags(q, count, host) {
Some(found) => not_null.extend(found),
None => pending = true,
}
// The outer columns' flags without the subquery items' pessimism, by projection position.
match outer_flags(q, host) {
Some(found) => not_null.extend(found),
Expand All @@ -671,6 +676,24 @@ fn prove(q: &Query, raw: Option<&MysqlDescribe>, views: &HashSet<String>, host:
Some(Proven { flags: described.columns.iter().map(|c| c.flags & 1 != 0).collect(), columns: described.columns, not_null, shaped })
}

/// The positions of the columns that are an aggregate called as a window function over a frame
/// that always holds the current row ([`crate::mysql_nulls::windowed`]) and are never NULL: a JSON
/// aggregate, and one that follows its argument where the copy of the statement that selects the
/// argument in its place flags it NOT NULL or the WHERE proves it. The server flags every such
/// call nullable but the bit functions. `count` is how many columns the query has.
fn window_flags(q: &Query, count: Option<usize>, host: &Host) -> Lazy<Vec<usize>> {
let Some((probe, follows, mut found)) = crate::mysql_nulls::window_probe(q) else { return Some(Vec::new()) };
if !follows.is_empty() {
if let Ok(d) = host.describe(&probe.to_string())? {
if Some(d.columns.len()) == count {
let tested = crate::mysql_nulls::tested(&probe);
found.extend(follows.into_iter().filter(|&i| d.columns.get(i).is_some_and(|c| c.flags & 1 != 0) || tested.contains(&i)));
}
}
}
Some(found)
}

/// The positions of the outer columns the statement without its subquery items flags NOT NULL
/// ([`crate::mysql_shape::outer_probe`]).
fn outer_flags(q: &Query, host: &Host) -> Lazy<Vec<usize>> {
Expand Down
196 changes: 196 additions & 0 deletions engine/mysql/src/mysql_nulls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,154 @@ fn nulls_with(e: &Expr) -> Vec<Vec<String>> {
}
}

/// The aggregates that, as window functions, are NULL only where no non-null value reaches the
/// frame: over a frame that holds the current row they are as NULL as their argument there.
const WINDOW_FOLLOWING: &[&str] = &["sum", "avg", "min", "max"];
/// The aggregates that are NULL only over no rows: over a frame that holds the current row they
/// are an array or an object, NULL inputs and all (a NULL key is an error).
const WINDOW_NEVER_NULL: &[&str] = &["json_arrayagg", "json_objectagg"];

/// What a window call says of its result column, where its frame always holds the current row.
#[derive(Debug, PartialEq)]
pub(crate) enum Windowed<'a> {
/// As NULL as this argument on the current row.
Follows(&'a Expr),
/// Never NULL.
Always,
}

/// What a select-list item of a plain SELECT says of its column when it is an aggregate called as
/// a window function whose frame always holds the current row ([`frame_holds_current_row`]):
/// `sum`, `avg`, `min` and `max` follow their argument, and `json_arrayagg` and `json_objectagg`
/// are never NULL. `None` for anything else, a call with DISTINCT or a FILTER among them.
pub(crate) fn windowed<'a>(e: &'a Expr, named: &[ast::NamedWindowDefinition]) -> Option<Windowed<'a>> {
let e = strip(e);
let Expr::Function(f) = e else { return None };
let over = f.over.as_ref()?;
if f.filter.is_some() || !f.within_group.is_empty() || f.null_treatment.is_some() {
return None;
}
let ast::FunctionArguments::List(list) = &f.args else { return None };
if list.duplicate_treatment.is_some() || !list.clauses.is_empty() {
return None;
}
let args: Vec<&Expr> = list
.args
.iter()
.map(|a| match a {
ast::FunctionArg::Unnamed(ast::FunctionArgExpr::Expr(x)) => Some(x),
_ => None,
})
.collect::<Option<_>>()?;
let name = crate::mysql::builtin(f);
if !frame_holds_current_row(over, named) {
return None;
}
match (name.as_str(), args.as_slice()) {
(n, [arg]) if WINDOW_FOLLOWING.contains(&n) => Some(Windowed::Follows(arg)),
("json_arrayagg", [_]) | ("json_objectagg", [_, _]) if WINDOW_NEVER_NULL.contains(&name.as_str()) => Some(Windowed::Always),
_ => None,
}
}

/// Whether the frame of an OVER clause always holds the current row, given the SELECT's WINDOW
/// clause. No frame clause is the whole partition, or with ORDER BY every row up to the current
/// row's last peer. A ROWS frame that starts UNBOUNDED PRECEDING, `n PRECEDING` or at CURRENT ROW
/// and ends at CURRENT ROW, `n FOLLOWING` or UNBOUNDED FOLLOWING holds it, and so does a RANGE
/// frame whose bounds are unbounded or CURRENT ROW. A start `n FOLLOWING` or an end `n PRECEDING`
/// can leave the row out, and so can a RANGE frame with an offset: measured on 8.4, over a
/// DATETIME that holds a zero date, `range between interval 1 day preceding and interval 1 day
/// following` finds no row for any row whose date is not NULL. A window named in the OVER clause,
/// alone or as the base of a definition, lends its frame where the definition has none of its own.
pub(crate) fn frame_holds_current_row(over: &ast::WindowType, named: &[ast::NamedWindowDefinition]) -> bool {
let mut spec = match over {
ast::WindowType::WindowSpec(spec) => spec,
ast::WindowType::NamedWindow(name) => match lookup(name, named) {
Some(spec) => spec,
None => return false,
},
};
// Every step names a window defined before it, so the walk ends; the bound guards a WINDOW
// clause the server would refuse.
for _ in 0..=named.len() {
if let Some(frame) = &spec.window_frame {
return holds_current_row(frame);
}
match &spec.window_name {
None => return true,
Some(name) => match lookup(name, named) {
Some(base) => spec = base,
None => return false,
},
}
}
false
}

/// The definition of the window of this name in a WINDOW clause, following a definition that is
/// another window's name alone (`w2 AS w1`).
fn lookup<'a>(name: &ast::Ident, named: &'a [ast::NamedWindowDefinition]) -> Option<&'a ast::WindowSpec> {
let mut name = name;
for _ in 0..=named.len() {
let ast::NamedWindowDefinition(_, expr) = named.iter().find(|ast::NamedWindowDefinition(n, _)| n.value.eq_ignore_ascii_case(&name.value))?;
match expr {
ast::NamedWindowExpr::WindowSpec(spec) => return Some(spec),
ast::NamedWindowExpr::NamedWindow(other) => name = other,
}
}
None
}

fn holds_current_row(frame: &ast::WindowFrame) -> bool {
use ast::{WindowFrameBound as B, WindowFrameUnits as U};
let offset = |b: &B| matches!(b, B::Preceding(Some(_)) | B::Following(Some(_)));
let start = matches!(frame.start_bound, B::Preceding(_) | B::CurrentRow);
let end = matches!(frame.end_bound, None | Some(B::CurrentRow | B::Following(_)));
let ranged = match frame.units {
U::Rows => true,
U::Range => !offset(&frame.start_bound) && !frame.end_bound.as_ref().is_some_and(offset),
U::Groups => false,
};
start && end && ranged
}

/// The statement with each select-list item that is a window call [`windowed`] finds following
/// its argument written as that argument, and the positions of those items and of the ones never
/// NULL: prepared, the copy's flag for such an item is the argument's on the current row, under
/// the same FROM, WHERE and GROUP BY. `None` for anything but a plain SELECT with such an item,
/// where a wildcard makes the positions unreadable, and under a ROLLUP, whose summary rows the
/// window reads too: the server's flags there are left as they are.
pub(crate) fn window_probe(q: &Query) -> Option<(Query, Vec<usize>, Vec<usize>)> {
let SetExpr::Select(select) = q.body.as_ref() else { return None };
if select.projection.iter().any(|i| matches!(i, SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(..))) || rolls_up(&select.group_by) {
Comment thread
pullfrog[bot] marked this conversation as resolved.
Resolved
return None;
}
let (mut follows, mut always) = (Vec::new(), Vec::new());
let mut projection = select.projection.clone();
for (i, item) in select.projection.iter().enumerate() {
let (SelectItem::UnnamedExpr(e) | SelectItem::ExprWithAlias { expr: e, .. }) = item else { continue };
match windowed(e, &select.named_window) {
Some(Windowed::Follows(arg)) => {
follows.push(i);
match &mut projection[i] {
SelectItem::UnnamedExpr(e) | SelectItem::ExprWithAlias { expr: e, .. } => *e = arg.clone(),
_ => {}
}
}
Some(Windowed::Always) => always.push(i),
None => {}
}
}
if follows.is_empty() && always.is_empty() {
return None;
}
let mut probe = q.clone();
if let SetExpr::Select(select) = probe.body.as_mut() {
select.projection = projection;
}
Some((probe, follows, always))
}

/// Whether a SELECT has a plain GROUP BY, where every group has a row.
pub(crate) fn grouped(group_by: &GroupByExpr) -> bool {
matches!(group_by, GroupByExpr::Expressions(exprs, _) if !exprs.is_empty()) && !rolls_up(group_by)
Expand Down Expand Up @@ -407,4 +555,52 @@ mod tests {
assert!(positions("select email from users where other.email > ''").is_empty());
assert!(positions("select upper(email) from users where email > ''").is_empty());
}

#[test]
fn a_window_aggregate_over_the_current_row_follows_its_argument() {
let probe = |sql: &str| window_probe(&query(sql)).map(|(q, follows, always)| (q.to_string(), follows, always));
let holds = |over: &str| probe(&format!("select sum(a) over {over} from t window w as (partition by a), v as (order by a rows between 1 following and 2 following)")).is_some();
// No frame clause, and a frame from at or before the current row to at or after it.
for over in [
"()",
"(order by a)",
"(partition by a)",
"(order by a rows between 1 preceding and 1 following)",
"(order by a rows unbounded preceding)",
"(order by a rows between current row and unbounded following)",
"(order by a rows between ? preceding and ? following)",
"(order by a range between unbounded preceding and current row)",
"(order by a range between current row and unbounded following)",
"w",
"(w order by a)",
"(w order by a rows between 1 preceding and current row)",
] {
assert!(holds(over), "{over}");
}
// A frame that can leave the current row out, a RANGE offset, and a window not defined.
for over in [
"(order by a rows between 1 following and 2 following)",
"(order by a rows between unbounded preceding and 1 preceding)",
"(order by a range between 1 preceding and 1 following)",
"(order by a range between interval 1 day preceding and current row)",
"(order by a range between current row and 0 following)",
"v",
"x",
"(x order by a)",
] {
assert!(!holds(over), "{over}");
}
// A named window lends its frame through a chain of definitions.
assert!(probe("select sum(a) over w2 from t window w1 as (partition by a), w2 as (w1 order by a)").is_some());
assert!(probe("select sum(a) over w3 from t window w1 as (order by a rows between 1 following and 1 following), w3 as (w1)").is_none());
// The argument takes the call's place, the JSON aggregates are never NULL, and the others
// prove nothing.
assert_eq!(probe("select id, sum(a) over () as s, max(b) over (order by id) from t"), Some(("SELECT id, a AS s, b FROM t".into(), vec![1, 2], vec![])));
assert_eq!(probe("select json_arrayagg(b) over () as j, json_objectagg(a, b) over () as o from t"), Some(("SELECT json_arrayagg(b) OVER () AS j, json_objectagg(a, b) OVER () AS o FROM t".into(), vec![], vec![0, 1])));
for item in ["sum(a)", "count(a) over ()", "var_samp(a) over ()", "group_concat(a)", "lag(a) over ()", "`sum`(a) over ()", "app.sum(a) over ()"] {
assert_eq!(probe(&format!("select {item} from t")), None, "{item}");
}
assert_eq!(probe("select *, sum(a) over () from t"), None, "a wildcard");
assert_eq!(probe("select a, sum(a) over () from t group by a with rollup"), None, "a ROLLUP");
}
}
17 changes: 12 additions & 5 deletions engine/mysql/src/mysql_shape.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ pub fn plan(q: &Query, columns: &Columns) -> Plan {
let hint = if crate::mysql_derived::is_materialized(q) { crate::mysql_derived::hint() } else { String::new() };
let grouped = crate::mysql_nulls::grouped(&select.group_by);
let top = Level { ctx: 0, from: select.from.clone(), with: q.with.clone() };
let mut b = Builder { stack: vec![top], columns, ctes: with_names(q), wanted: Vec::new(), next_ctx: 0, drafts: Vec::new(), overflowed: false, booleans: Vec::new(), either: Vec::new(), reads: Reads::Itself, grouped };
let mut b = Builder { stack: vec![top], columns, ctes: with_names(q), wanted: Vec::new(), next_ctx: 0, drafts: Vec::new(), overflowed: false, booleans: Vec::new(), either: Vec::new(), reads: Reads::Itself, grouped, windows: select.named_window.clone() };
let mut columns = Vec::new();
for item in &select.projection {
// The server names an unaliased expression by its text, which no build reproduces
Expand Down Expand Up @@ -360,6 +360,8 @@ struct Builder<'a> {
reads: Reads,
/// Whether the statement's SELECT has a plain GROUP BY, where no group is empty.
grouped: bool,
/// The statement's WINDOW clause, which an OVER clause of its own SELECT names windows of.
windows: Vec<ast::NamedWindowDefinition>,
}

/// How a JSON function reads the value it writes, which decides whether a truth value is
Expand Down Expand Up @@ -530,14 +532,19 @@ impl Builder<'_> {
Shape::Object { fields, non_null: true }
}
// An aggregate over no rows is NULL, so the array is non-null only under a
// guard, or under a plain GROUP BY, where every group has a row, for an
// aggregate of the statement's own SELECT that is no window function. One
// inside a subquery may be the outer query's, which says nothing of its rows.
// guard, under a plain GROUP BY, where every group has a row, or over a window
// frame that always holds the current row, for an aggregate of the statement's
// own SELECT. One inside a subquery may be the outer query's, which says
// nothing of its rows.
"json_arrayagg" => {
if args.len() != 1 {
return Shape::Unknown;
}
let non_null = self.grouped && self.stack.len() == 1 && f.over.is_none();
let has_a_row = match &f.over {
None => self.grouped,
Some(_) => crate::mysql_nulls::windowed(e, &self.windows) == Some(crate::mysql_nulls::Windowed::Always),
};
let non_null = has_a_row && self.stack.len() == 1;
Shape::Array { items: Box::new(self.walk_as(args[0], Reads::Copy)), non_null }
}
// The value is the first argument that is not NULL. A constructor that is
Expand Down
4 changes: 2 additions & 2 deletions oracle/cases/mysql_json_shapes/expected.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ array_agg_truth_buffered.sql: one () -> { l: [bool|int8]? }
array_agg_truth_cast.sql: many () -> { l: [bool] }
array_agg_truth_join.sql: many () -> { l: [bool|int8] }
array_agg_truth_object.sql: many () -> { l: [{ t: bool }] }
array_agg_truth_window.sql: many () -> { l: [bool|int8]? }
array_agg_window.sql: many () -> { ids: [int4]? }
array_agg_truth_window.sql: many () -> { l: [bool|int8] }
array_agg_window.sql: many () -> { ids: [int4] }
cast_as_json.sql: many () -> { o: { id: int4 }, n: json? }
json_array_literal.sql: many () -> { a: json?, none: json? }
json_functions.sql: many () -> { a: json?, b: json?, c: text?, merged: json?, changed: json?, removed: json?, k: json?, q: text?, u: text? }
Expand Down
2 changes: 1 addition & 1 deletion oracle/cases/mysql_row_counts/expected.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
aggregate_and_window.sql: one () -> { n: int8, total: numeric? }
aggregate_and_window.sql: one () -> { n: int8, total: numeric }
aggregate_any_value.sql: many () -> { name: text }
aggregate_bit.sql: one () -> { a: int8, o: int8, x: int8 }
aggregate_distinct.sql: one () -> { n: int8 }
Expand Down
4 changes: 2 additions & 2 deletions oracle/cases/mysql_server_flags/expected.txt
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,6 @@ view_columns.sql: many () -> { id: int4|int8, n: int4|int8, more: int8|int4, m:
view_columns_ordered.sql: many () -> { id: int4|int8, n: int4|int8, more: int8|int4, m: int8, small: int2, q: int8|int4?, big: int8 }
view_expression_alone.sql: many () -> { more: int8|int4, q: int8|int4? }
where_narrows.sql: many () -> { maybe_name: text, maybe_n: int4 }
window_functions.sql: many () -> { a: int8, b: int8, c: int8, d: int8, e: numeric?, f: int4|int8?, g: text?, h: text?, i: int4|int8?, j: int4|int8?, k: int8, l: float8, m: float8, o: int4|int8?, p: int8, q: text?, r: numeric?, s: int8, u: numeric?, v: int4|int8?, w: int8, x: [int4]? }
window_functions.sql: many () -> { a: int8, b: int8, c: int8, d: int8, e: numeric, f: int4|int8?, g: text?, h: text?, i: int4|int8?, j: int4|int8?, k: int8, l: float8, m: float8, o: int4|int8, p: int8, q: text, r: numeric, s: int8, u: numeric?, v: int4|int8?, w: int8, x: [int4] }
with_query.sql: many () -> { name: text, maybe_name: text?, next: int8 }
zero_dates.sql: many () -> { a: date, b: date?, c: int2?, d: int8|int4?, e: int8, f: int8, g: text?, i: text?, j: text?, k: date?, l: text, m: date, o: bool, p: bool, q: int8|int4, r: text, s: timestamp, u: date?, v: timestamp?, w: timestamp, x: date?, z: bool, aa: int8|int4?, ab: int8|int4?, ac: int8|int4?, ad: int8|int4?, ae: text?, af: date?, ag: date?, ah: date?, ai: date?, aj: int8|int4?, ak: int8? }
zero_dates.sql: many () -> { a: date, b: date?, c: int2?, d: int8|int4?, e: int8, f: int8, g: text?, i: text?, j: text?, k: date?, l: text, m: date, o: bool, p: bool, q: int8|int4, r: text, s: timestamp, u: date?, v: timestamp?, w: timestamp, x: date, z: bool, aa: int8|int4?, ab: int8|int4?, ac: int8|int4?, ad: int8|int4?, ae: text?, af: date?, ag: date?, ah: date?, ai: date?, aj: int8|int4?, ak: int8? }
5 changes: 3 additions & 2 deletions E551 oracle/cases/mysql_server_flags/setup.sql
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
-- Nullability is the server's NOT NULL flag, which the engine takes as it is: sound where it is
-- set, and nullable for string and date functions and most casts of NOT NULL inputs. Only what the
-- statement proves narrows it: a strict test in the WHERE (`engine/mysql/src/mysql_nulls.rs`)
-- proves the column it tests and the row of an outer join, and a GROUP BY's JSON_ARRAYAGG is
-- never NULL. Every
-- proves the column it tests and the row of an outer join, a GROUP BY's JSON_ARRAYAGG is never
-- NULL, and an aggregate over a window frame that holds the current row is as NULL as its argument
-- (`window_flags` in `engine/mysql/src/mysql.rs`). Every
-- column here is NOT NULL but those named `maybe_`, and the edge state holds zero dates, the
-- bounds of every integer and an empty string.
create table t (
Expand Down
2 changes: 1 addition & 1 deletion oracle/cases/mysql_types/expected.txt
Original file line number Diff line number Diff line change
Expand Up @@ -92,5 +92,5 @@ unsigned_overflow.sql: many () -> { past: int8?, under: int8|int4? }
update_by_key.sql: none (body: text, n: int4, id: int4) -> { }
update_join.sql: none (v: text) -> { }
update_limit.sql: none (score: int4, n: int4) -> { }
window.sql: many () -> { rn: int8, rk: int8, total: numeric?, previous: int4|int8?, fv: text?, nt: int8, pr: float8, n: int8 }
window.sql: many () -> { rn: int8, rk: int8, total: numeric, previous: int4|int8?, fv: text?, nt: int8, pr: float8, n: int8 }
with_insert.sql: none (body: text) -> { }
Loading
Loading
0