8000 fix: report odd json object arguments and errors at end of input by colinhacks · Pull Request #7 · colinhacks/reylo · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 112 additions & 0 deletions engine/core/src/analysis.rs
10BC8
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,81 @@ pub fn repeated(sql: &str, lexed: &Lexed) -> Vec<Diagnostic> {
lex::repeated(sql, lexed).into_iter().map(|r| diag(r.message, r.start, r.end - r.start, 3)).collect()
}

/// Each call of one of `names` written with an odd number of arguments: the constructors of a JSON
/// object take keys and values in pairs, and a database prepares a call with an odd count and
/// refuses it only when it runs. Read from the text outside the lexer's strings and comments
/// ([`Lexed::opaque`]); a name qualified with a schema is somebody else's function, and a
/// `VARIADIC` argument is an array whose length the text does not say.
pub fn odd_pairs(sql: &str, lexed: &Lexed, names: &[&str]) -> Vec<Diagnostic> {
let bytes = sql.as_bytes();
let opaque_at = |i: usize| lexed.opaque.iter().find(|(s, e)| *s <= i && i < *e).map(|(_, e)| *e);
// Postgres and SQLite read every byte past ASCII as part of a name (`café`, `éjson_build_object`).
// The scan steps byte by byte and compares bytes, so it never slices inside a character.
let is_name = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b'$' || b >= 0x80;
let mut found = Vec::new();
let mut i = 0;
while i < bytes.len() {
if let Some(end) = opaque_at(i) {
i = end;
continue;
}
let starts_word = is_name(bytes[i]) && (i == 0 || !(is_name(bytes[i - 1]) || bytes[i - 1] == b'.'));
let Some(name) = names.iter().find(|n| starts_word && bytes.get(i..i + n.len()).is_some_and(|w| w.eq_ignore_ascii_case(n.as_bytes())) && !bytes.get(i + n.len()).is_some_and(|b| is_name(*b))) else {
i += 1;
continue;
};
let mut j = i + name.len();
while j < bytes.len() && bytes[j].is_ascii_whitespace() {
j += 1;
}
if bytes.get(j) != Some(&b'(') {
i += name.len();
continue;
}
// The arguments: the commas at the call's own depth, past every string and comment.
let (mut depth, mut commas, mut empty, mut variadic, mut k) = (0usize, 0usize, true, false, j);
while k < bytes.len() {
if let Some(end) = opaque_at(k) {
if depth == 1 {
empty = false;
}
k = end;
continue;
}
match bytes[k] {
b'(' | b'[' => {
if depth == 1 {
empty = false;
}
depth += 1;
}
b')' | b']' => {
depth -= 1;
if depth == 0 {
break;
}
}
b',' if depth == 1 => commas += 1,
b if depth == 1 && !b.is_ascii_whitespace() => {
empty = false;
if !is_name(bytes[k - 1]) && bytes.get(k..k + 8).is_some_and(|w| w.eq_ignore_ascii_case(b"variadic")) && !bytes.get(k + 8).is_some_and(|b| is_name(*b)) {
variadic = true;
}
}
_ => {}
}
k += 1;
}
let count = if empty { 0 } else { commas + 1 };
if k < bytes.len() && !variadic && count % 2 == 1 {
let written = &sql[i..i + name.len()];
found.push(diag(format!("{written} takes keys and values in pairs, and this call has {count} argument{}, which the database refuses when it runs.", if count == 1 { "" } else { "s" }), i, name.len(), 15));
}
i = j + 1;
}
found
}

/// A diagnostic at least one byte long.
pub fn diag(message: impl Into<String>, start: usize, length: usize, code: u32) -> Diagnostic {
Diagnostic { message: message.into(), start, length: length.max(1), code }
Expand Down Expand Up @@ -65,3 +140,40 @@ pub fn bare_of(sql: &str, dialect: Dialect, diagnostics: Vec<Diagnostic>) -> Ana
let lexed = lex::lex(sql, dialect).unwrap_or_else(|_| none());
bare(sql, &lexed, dialect, diagnostics)
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn a_json_object_of_an_odd_count_of_arguments_is_an_error_at_its_name() {
let names = ["json_build_object", "jsonb_build_object"];
let at = |sql: &str| {
let lexed = lex::lex(sql, Dialect::Postgres).unwrap();
odd_pairs(sql, &lexed, &names).iter().map(|d| (sql[d.start..d.start + d.length].to_string(), d.message.clone(), d.code)).collect::<Vec<_>>()
};
let found = at("select json_build_object('a', 1, 'b'), JSONB_BUILD_OBJECT('k')");
assert_eq!(found.iter().map(|(w, _, code)| (w.as_str(), *code)).collect::<Vec<_>>(), [("json_build_object", 15), ("JSONB_BUILD_OBJECT", 15)]);
assert_eq!(found[0].1, "json_build_object takes keys and values in pairs, and this call has 3 arguments, which the database refuses when it runs.");
assert!(found[1].1.contains("this call has 1 argument,"));
// Pairs, nested calls, commas inside strings, arrays and comments, no arguments at all,
// a VARIADIC array, another schema's function and a name inside a string are all fine.
for sql in [
"select json_build_object('a', f(1, 2), 'b,c', array[1, 2])",
"select json_build_object()",
"select json_build_object( )",
"select jsonb_build_object(variadic array['a', 'b'])",
"select app.json_build_object('a')",
"select 'json_build_object(1)' as t",
"select json_build_object('a', /* , */ 1)",
"select json_build_object('a', café, 'b', 1)",
"select éjson_build_object('a')",
"select json_build_objecté('a')",
] {
assert!(at(sql).is_empty(), "{sql}");
}
// A name past ASCII among the arguments is one argument, and the scan never splits it.
assert_eq!(at("select json_build_object('a', café, 'b')").len(), 1);
assert_eq!(at("select json_build_object(variadicé, 'a', 1)").len(), 1);
}
}
56 changes: 49 additions & 7 deletions engine/postgres/src/postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,10 @@ impl<'a> Engine<'a> {
diagnostics.push(diag(message, start, length, 13));
}
}
// A JSON object of an odd count of arguments, which Postgres prepares and refuses when it
// runs; only the built-in constructors, which a user's function of the name may shadow.
let constructors: Vec<&str> = ["json_build_object", "jsonb_build_object"].into_iter().filter(|n| trust.builtin(n)).collect();
diagnostics.extend(crate::analysis::odd_pairs(sql, lexed, &constructors));

// JSON shapes: what the statement's own constructors build, and for a JSON column it reads
// straight from a WITH query or a derived table, what that item's statement builds.
Expand Down Expand Up @@ -1882,13 +1886,19 @@ fn build_catalog(operators: &HashSet<String>, rows: CatalogRows) -> Catalog {
/// A failed describe as a diagnostic on the author's text: the server's message, at the
/// characters its position points at.
fn db_diagnostic(sql: &str, rewritten: &Rewritten, e: &DbError) -> Diagnostic {
let start = e
.position
.filter(|p| *p > 0)
.and_then(|p| rewritten.sql.char_indices().nth(p - 1).map(|(b, _)| b))
.map(|b| rewritten.to_original(b))
.unwrap_or(0)
.min(sql.len());
let at = e.position.filter(|p| *p > 0).map(|p| rewritten.sql.char_indices().nth(p - 1).map_or(sql.len(), |(b, _)| rewritten.to_original(b).min(sql.len())));
let start = match at {
// A position at the end of the text, or in the whitespace after it, is the end of input
// ("syntax error at end of input"): the statement's last word, where it stops short.
// Postgres's whitespace is ASCII alone (a no-break space is part of a name to it), which
// also keeps the start after the separator on a character's boundary.
Some(at) if sql[at..].trim_matches(is_space).is_empty() => {
let end = sql.trim_end_matches(is_space).len();
sql[..end].rfind(|c: char| is_space(c) || matches!(c, '(' | ',' | ';')).map_or(0, |i| i + 1).min(end.saturating_sub(1))
}
Some(at) => at,
None => 0,
};
let length = sql[start..].bytes().take_while(|c| c.is_ascii_alphanumeric() || matches!(c, b'_' | b'.' | b'"' | b'$' | b':' | b'@')).count();
let message = match &e.code {
Some(code) => format!("{} ({code})", e.message),
Expand All @@ -1897,6 +1907,12 @@ fn db_diagnostic(sql: &str, rewritten: &Rewritten, e: &DbError) -> Diagnostic {
diag(message, start, length, 6)
}

/// Whitespace as Postgres's scanner reads it: space, tab, newline, carriage return, form feed and
/// vertical tab.
fn is_space(c: char) -> bool {
matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0c' | '\x0b')
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down Expand Up @@ -2467,4 +2483,30 @@ mod tests {
// Two may be two columns and none, so no position says which column a target is.
assert_eq!(own(&format!("{with} SELECT a.*, u.name, e.* FROM a, users u, e"), 3), vec![false; 3]);
}

#[test]
fn an_error_at_the_end_of_input_is_on_the_last_word() {
for sql in ["select id from member where\n", "select id from member where", "select id from t where id = :id and\n\n"] {
let lexed = lex::lex(sql, Dialect::Postgres).unwrap();
let rewritten = lex::rewrite(sql, &lexed, Target::Dollar);
let last = sql.trim_end().rsplit(' ').next().unwrap();
for position in [rewritten.sql.trim_end().chars().count() + 1, rewritten.sql.chars().count() + 1] {
let e = DbError { message: "syntax error at end of input".into(), code: Some("42601".into()), position: Some(position) };
let d = db_diagnostic(sql, &rewritten, &e);
assert_eq!(&sql[d.start..d.start + d.length], last, "{sql:?} at {position}");
}
}
// A no-break space is part of a name to Postgres, and the start never lands inside one.
let sql = "select 1 +\u{a0}-";
let lexed = lex::lex(sql, Dialect::Postgres).unwrap();
let rewritten = lex::rewrite(sql, &lexed, Target::Dollar);
let d = db_diagnostic(sql, &rewritten, &DbError { message: "syntax error at end of input".into(), code: Some("42601".into()), position: Some(sql.chars().count() + 1) });
assert_eq!(&sql[d.start..d.start + d.length], "+");
// A position inside the text stays where it is.
let sql = "select idd from t";
let lexed = lex::lex(sql, Dialect::Postgres).unwrap();
let rewritten = lex::rewrite(sql, &lexed, Target::Dollar);
let d = db_diagnostic(sql, &rewritten, &DbError { message: "column \"idd\" does not exist".into(), code: None, position: Some(8) });
assert_eq!(&sql[d.start..d.start + d.length], "idd");
}
}
18 changes: 18 additions & 0 deletions engine/sqlite/src/sqlite.rs
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,12 @@ fn run(sql: &str, lexed: &Lexed, ddl: &[String], extras: &Extras) -> SqliteAnaly
}
return failed(bare(sql, lexed, Dialect::Sqlite, diagnostics));
}
// A JSON object of an odd count of arguments, which SQLite prepares and refuses when it runs;
// only the built-in constructors, which a function the host registers under the name overrides
// (`Extras::function`). The host's list names the built-ins with any argument count (-1).
let registered = |name: &str| extras.total.iter().any(|f| f.name.eq_ignore_ascii_case(name)) || extras.host.iter().flatten().any(|f| f.name.eq_ignore_ascii_case(name) && f.arity >= 0);
let constructors: Vec<&str> = ["json_object", "jsonb_object"].into_iter().filter(|n| !registered(n)).collect();
diagnostics.extend(crate::analysis::odd_pairs(sql, lexed, &constructors));
let a = Analyzer { tree: &t, extras, skipping: &skipping, nullable_params: coalesced_params(&t, extras), level: std::cell::RefCell::new(Level::default()), shapes: std::cell::RefCell::new(HashMap::new()) };
let (mut columns, kind) = a.columns();
let booleans = a.booleans();
Expand Down Expand Up @@ -4005,6 +4011,18 @@ mod tests {
assert_eq!(a.params.iter().map(|p| format!("{} {} {}", p.name, ts(&p.ty, false), p.type_name)).collect::<Vec<_>>(), ["doc string text", "a unknown ", "j string text"]);
}

#[test]
fn a_json_object_of_an_odd_count_of_arguments_is_built_in_alone() {
let sql = "select json_object('id', id, 'name') as j from users";
let odd = |extras: &Extras| analyze_ddl(sql, &ddl(), extras).analysis.diagnostics.iter().filter(|d| d.code == 15).count();
assert_eq!(odd(&Extras::default()), 1);
let listed = |arities: &[i32]| Extras { total: vec![], host: Some(arities.iter().map(|&arity| HostFunction { name: "json_object".into(), arity, kind: 's' }).collect()) };
assert_eq!(odd(&listed(&[-1])), 1);
// A function of the name the host registers is called in the built-in's place.
assert_eq!(odd(&listed(&[-1, 3])), 0);
assert_eq!(odd(&Extras { total: vec![TotalFunction { name: "JSON_OBJECT".into(), arity: 3, returns: Affinity::Text }], host: None }), 0);
}

#[test]
fn sqlite_only_syntax_parses() {
let a = analysis("select id from users limit :n, :m");
Expand Down
1 change: 1 addition & 0 deletions oracle/cases/odd_json_pairs/ends_early.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select id from t where
3 changes: 3 additions & 0 deletions oracle/cases/odd_json_pairs/expected.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
ends_early.sql: many () -> { } !! reylo6: syntax error at end of input (42601)
odd.sql: many () -> { j: json } !! reylo15: json_build_object takes keys and values in pairs, and this call has 3 arguments, which the database refuses when it runs.
pairs.sql: many () -> { j: { id: int4, name: text } }
1 change: 1 addition & 0 deletions oracle/cases/odd_json_pairs/odd.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select json_build_object('id', id, 'name') as j from t
1 change: 1 addition & 0 deletions oracle/cases/odd_json_pairs/pairs.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select jsonb_build_object('id', id, 'name', name) as j from t
3 changes: 3 additions & 0 deletions oracle/cases/odd_json_pairs/setup.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- A JSON object built from an odd count of arguments, which Postgres prepares and refuses when
-- it runs, and a statement that ends too soon.
create table t (id int primary key, name text not null);
2 changes: 2 additions & 0 deletions oracle/cases/odd_json_pairs/unchecked.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
ends_early.sql: syntax error at end of input (42601)
odd.sql: argument list must have even number of elements (22023)
1 change: 1 addition & 0 deletions oracle/cases/sqlite_odd_json_pairs/dialect
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
sqlite
2 changes: 2 additions & 0 deletions oracle/cases/sqlite_odd_json_pairs/expected.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
odd.sql: many () -> { j: json } !! reylo15: json_object takes keys and values in pairs, and this call has 3 arguments, which the database refuses when it runs.
pairs.sql: many () -> { j: { id: integer, name: text } }
1 change: 1 addition & 0 deletions oracle/cases/sqlite_odd_json_pairs/odd.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select json_object('id', id, 'name') as j from t
1 change: 1 addition & 0 deletions oracle/cases/sqlite_odd_json_pairs/pairs.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
select json_object('id', id, 'name', name) as j from t
3 changes: 3 additions & 0 deletions oracle/cases/sqlite_odd_json_pairs/setup.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- A JSON object built from an odd count of arguments, which SQLite prepares and refuses when it
-- runs.
create table t (id integer primary key, name text not null);
1 change: 1 addition & 0 deletions oracle/cases/sqlite_odd_json_pairs/unchecked.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
odd.sql: json_object() requires an even number of arguments
Loading
Loading
0