FFFF feat(mysql): type vector columns and parameters as number[] by colinhacks · Pull Request #8 · colinhacks/reylo · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions engine/mysql/src/mysql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,7 @@ pub fn analyze(sql: &str, lexed: &Lexed, text: Option<MysqlText>, host: &Host) -
None if raw.columns.is_empty() && opens_without_result_sets(&rewritten.sql) => ("none".to_string(), None),
None => ("many".to_string(), Some(misread.unwrap_or_else(|| "the engine could not parse the statement, so nothing bounds its rows".into()))),
};
let rewritten = vectors_as_text(rewritten, &raw);
Some(Analysis {
kind: kind.to_string(),
params,
Expand Down Expand Up @@ -1041,6 +1042,10 @@ fn column_type(code: u32, charset: u32) -> (Type, &'static str) {
10 | 14 => (scalar("date"), "date"),
11 => (scalar("time"), "time"),
245 => (Type::Json { schema: None }, "json"),
// MySQL 9's VECTOR, a column and the context of a `?` alike: 32-bit floats, none of them
// NULL, whose value comes as their little-endian bytes (measured on 9.7.2). A parameter
// binds their text instead (`vectors_as_text`).
242 => (Type::Array { of: Box::new(scalar("float4")), nullable_items: Some(false) }, "vector"),
16 => (scalar("bytea"), "bit"),
249..=254 | 15 => {
if charset == 63 { (scalar("bytea"), "binary") } else { (scalar("text"), "varchar") }
Expand All @@ -1050,6 +1055,37 @@ fn column_type(code: u32, charset: u32) -> (Type, &'static str) {
}
}

/// Each `?` the prepare typed as a VECTOR wrapped in `to_vector(…)`, so that it binds the text of
/// its floats (`[1,2.5]`), which the server reads as a VECTOR wherever it stands. A VECTOR's bytes
/// would not do on every driver: mariadb and PlanetScale write every value into the statement's
/// text, where bytes are a binary string, which MySQL takes for a VECTOR column or a function's
/// argument but refuses beside `=` ("Incorrect arguments to =", 9.7.2). `to_vector` is
/// `string_to_vector`'s other name.
fn vectors_as_text(rewritten: lex::Rewritten, raw: &MysqlDescribe) -> lex::Rewritten {
if !raw.params.iter().any(|p| p.r#type == 242) {
return rewritten;
}
let mut sql = String::with_capacity(rewritten.sql.len() + 11 * raw.params.len());
let mut spans = Vec::with_capacity(rewritten.spans.len());
let mut from = 0;
for (i, &(os, oe, rs, re)) in rewritten.spans.iter().enumerate() {
sql.push_str(&rewritten.sql[from..rs]);
let start = sql.len();
let vector = raw.params.get(i).is_some_and(|p| p.r#type == 242);
if vector {
sql.push_str("to_vector(");
}
sql.push_str(&rewritten.sql[rs..re]);
if vector {
sql.push(')');
}
spans.push((os, oe, start, sql.len()));
from = re;
}
sql.push_str(&rewritten.sql[from..]);
lex::Rewritten { sql, order: rewritten.order, spans }
}

/// A parameter's type from the server's context guess. Every integer context comes back as
/// LONGLONG, whatever the column's width, so an integer parameter is a plain number (the run
/// time's parameter types widen it to the BIGINT and TINYINT(1) values a row returns). A BIT
Expand Down Expand Up @@ -2068,6 +2104,25 @@ mod tests {
assert_eq!(types, ["b=Scalar { name: \"bool|bytea\" }", "r=Scalar { name: \"bytea\" }", "n=Scalar { name: \"int4\" }"]);
}

#[test]
fn a_vector_is_an_array_of_floats() {
// MySQL 9.7.2 describes a VECTOR(3) column, a NULL one, and a `?` compared with one, as type 242.
let described = json!({ "columns": [{ "name": "v", "type": 242, "flags": 4241, "charset": 63, "length": 12 }, { "name": "vn", "type": 242, "flags": 144, "charset": 63, "length": 8 }], "params": [{ "type": 242, "flags": 128, "charset": 63 }] });
let (analysis, _) = analyzed("select v, vn from items where v = :v", described, "STRICT_TRANS_TABLES");
let floats = Type::Array { of: Box::new(Type::Scalar { name: "float4".into() }), nullable_items: Some(false) };
let columns: Vec<_> = analysis.columns.iter().map(|c| (c.ty.clone(), c.nullable, c.type_name.as_str())).collect();
assert_eq!(columns, [(floats.clone(), false, "vector"), (floats.clone(), true, "vector")]);
assert_eq!(analysis.params[0].ty, floats);
// The parameter binds its floats' text, wrapped where the server reads it as a VECTOR; the
// map back to the author's text keeps the placeholder's own place.
assert_eq!(analysis.rewritten.sql, "select v, vn from items where v = to_vector(?)");
assert_eq!(analysis.rewritten.spans, [(34, 36, 34, 46)]);
let described = json!({ "columns": [], "params": [{ "type": 242 }, { "type": 242 }, { "type": 3 }, { "type": 3 }] });
let (analysis, _) = analyzed("update items set v = :v, vn = :w where id = :id and n = :n", described, "STRICT_TRANS_TABLES");
assert_eq!(analysis.rewritten.sql, "update items set v = to_vector(?), vn = to_vector(?) where id = ? and n = ?");
assert_eq!(analysis.rewritten.to_original(analysis.rewritten.sql.find("id = ?").unwrap() + 5), "update items set v = :v, vn = :w where id = :id and n = :n".find(":id").unwrap());
}

#[test]
fn placeholders_are_read_as_the_session_reads_strings() {
let by_id = json!({ "columns": [{ "name": "id", "type": 3, "flags": 1 }], "params": [{ "type": 3 }] });
Expand Down
Binary file modified packages/mysql/engine.wasm
Binary file not shown.
24 changes: 24 additions & 0 deletions site/content/docs/mysql.mdx
10BC8
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ Reylo decodes every value of its own queries itself, on every driver, so each SQ
| `json_object(…)`, `json_arrayagg(…)` | the object or array it builds | `{ total: 12.5 }` |
| `BINARY`, `VARBINARY`, `BLOB`, `BIT` wider than 1 | `Uint8Array` | `Uint8Array.of(1, 2, 255)` |
| `GEOMETRY` | `unknown` | its bytes, a `Uint8Array` |
| `VECTOR` | `number[]` | `[1, 2.5, -3]` |

### Integers

Expand Down Expand Up @@ -444,6 +445,28 @@ Compared with a value that is no label, an `ENUM` matches nothing, and an insert

</Rules>

### Vectors

A `VECTOR`, which MySQL 9 added, is a `number[]` of its 32-bit floats.

<Pairs>

- `STRING_TO_VECTOR('[1, 2.5, 0.1]')` → `[1, 2.5, 0.10000000149011612]`

</Pairs>

<Rules>

**A vector parameter**

A vector parameter takes a `number[]`, which Reylo sends as its text through `to_vector(?)`, so the parameter works wherever MySQL takes a `VECTOR`: written to a column, passed to a function, or compared with `=`. Reylo refuses `NaN` and the infinities before the statement runs, and MySQL refuses a number too large for a 32-bit float.

**On `Bun.sql`**

Bun 1.4 can't read a `VECTOR`: a statement that returns one fails inside Bun with `ERR_MYSQL_UNSUPPORTED_COLUMN_TYPE`. Select `vector_to_string(v)` instead, or run the statement on `mysql2`.

</Rules>

### Values Reylo refuses

Each driver hands Reylo exact integers and dates as text, so one value is refused, with an error that names its column:
Expand Down Expand Up @@ -561,6 +584,7 @@ Every value `.run()` returns goes back in as a parameter of its type, on every d
| `DATE`, `DATETIME`, `TIMESTAMP`, `TIME` | `string` |
| `JSON` | any JSON value |
| `BINARY`, `VARBINARY`, `BLOB` | `Uint8Array` |
| `VECTOR` | `number[]` |

- MySQL describes every integer parameter as a `BIGINT`, and every `BIT` one alike. So where the statement names the column, Reylo types a parameter by the column it is compared with or written to:
- in a comparison such as `col = :p`, an `in` or a `between`;
Expand Down
1 change: 1 addition & 0 deletions site/content/sections/07-data-types.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ Reylo decodes its own queries' values itself, so every driver gives the same typ
| `TIME` | `string` |
| `JSON` | [typed by its schema or the query](#json-values) |
| `BINARY`, `VARBINARY`, `BLOB`, `BIT` wider than 1 | `Uint8Array` |
| `VECTOR` | `number[]` |

</Dialect>
<Dialect value="SQLite">
Expand Down
7 changes: 6 additions & 1 deletion src/decode-mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,18 @@ const toBool: Fix = truth;
// A BIGINT's text is always its digits: BigInt reads it straight.
const toBigInt: Fix = (v, name) => (typeof v === "string" ? BigInt(v) : typeof v === "number" || typeof v === "bigint" ? bigInteger(v, name) : v);

// A VECTOR is MySQL's one array: mysql2 decodes its floats, mariadb and PlanetScale give its bytes.
const toFloats: Fix = (v, name) => coerce("float[]", v, name);

/**
* The fix a column of `kind` needs, if any: `bool` a boolean, `int8` (a BIGINT, or an integer a
* temporary table may widen to one) a BigInt from whatever the driver gave, a SET's members joined.
* temporary table may widen to one) a BigInt from whatever the driver gave, `float[]` (a VECTOR)
* its floats.
*/
function byKind(kind: Kind | undefined): Fix | undefined {
if (kind === "bool") return toBool;
if (kind === "int8") return toBigInt;
if (kind === "float[]") return toFloats;
return undefined;
}

Expand Down
23 changes: 18 additions & 5 deletions src/execute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ function statement(query: QueryModule, params: ParamsInput) {
const cast = query.inputs?.[name]?.cast;
if (cast !== undefined && type !== undefined) checkCast(name, cast, type, bound.values[i], query.dialect);
checkTime(type, name, bound.values[i]);
bound.values[i] = encode(type, bound.values[i], query.dialect);
bound.values[i] = encode(type, bound.values[i], query.dialect, name);
});
// Every MySQL and SQLite driver binds JSON text as text, which the database reads as JSON (a
// structured parameter's is written already).
Expand Down Expand Up @@ -456,15 +456,28 @@ function checkTime(type: EngineType | undefined, name: string, v: unknown): void
// on Postgres a date or time with an extended or BC year in Postgres's own form, which refuses
// `+010000` and `0000`; on SQLite a truth value, which a BOOLEAN column returns, as 1 or 0, since
// better-sqlite3 refuses a boolean ("SQLite3 can only bind numbers, strings, bigints, buffers, and
// null") and libSQL binds `true` as the REAL 1.0. A query without `inputs` gets neither.
function encode(type: EngineType | undefined, v: unknown, dialect: Dialect): unknown {
// null") and libSQL binds `true` as the REAL 1.0. A query without `inputs` gets neither. On MySQL
// a VECTOR, MySQL's one array, as the text of its floats, `[1,2.5]`, which the engine's
// `to_vector(?)` reads: drivers left to themselves sent an array as the text "1,2,3" (mysql2's
// execute()) or expanded it into a list of values (mysql2's query(), mariadb).
function encode(type: EngineType | undefined, v: unknown, dialect: Dialect, name: string): unknown {
if (type === undefined || v == null) return v;
if (dialect === "sqlite") return typeof v === "boolean" ? (v ? 1 : 0) : v;
if (dialect !== "postgres") return v;
if (type.kind === "array") return Array.isArray(v) ? v.map((e) => encode(Array.isArray(e) ? type : type.of, e, dialect)) : v;
if (dialect === "mysql") return type.kind === "array" && Array.isArray(v) ? vectorText(v, name) : v;
if (type.kind === "array") return Array.isArray(v) ? v.map((e) => encode(Array.isArray(e) ? type : type.of, e, dialect, name)) : v;
return typeof v === "string" && isTime(type) ? postgresYear(v) : v;
}

/** A VECTOR's floats as the text `to_vector` reads; JSON's, which would write NaN and ±Infinity as null, so those are refused. */
function vectorText(v: readonly unknown[], name: string): string {
v.forEach((e, i) => {
if (typeof e !== "number" || !Number.isFinite(e)) {
throw new TypeError(`Parameter "${name}" is a VECTOR, an array of finite numbers, and the element at [${i}] is ${typeof e === "string" ? JSON.stringify(e) : typeof e === "number" ? e : typeof e}.`);
}
});
return JSON.stringify(v);
}

// sqlite3 6.0.1 binds a BigInt as NULL, with no error (`select cast(? as text)` with
// 9007199254740993n gave null), where every other SQLite driver binds it exactly. A BigInt a
// number holds exactly goes as that number. A larger one goes as its digits to a parameter of a
Expand Down
12 changes: 12 additions & 0 deletions src/values.ts
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,16 @@ function utcText(d: Date, kind: "date" | "timestamp", column: string): string {

const isBytes = (v: unknown): v is Uint8Array => v instanceof Uint8Array;

/**
* A MySQL VECTOR's floats from its bytes, 32-bit and little-endian, which mariadb and PlanetScale
* hand over as they are (mysql2 decodes them itself).
*/
export function float32s(bytes: Uint8Array, column: string): number[] {
if (bytes.byteLength % 4 !== 0) throw new DecodeError(`Column "${column}" is a VECTOR, whose value is 4 bytes per 32-bit float, and holds ${bytes.byteLength} bytes.`);
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
return Array.from({ length: bytes.byteLength / 4 }, (_, i) => view.getFloat32(i * 4, true));
}

/**
* `v` as a value of `kind`, from the form a driver decoded it to; `column` names it in an error. A
* number past ±(2^53 - 1) in an integer column is one the driver already rounded, and is refused.
Expand All @@ -430,6 +440,8 @@ export function coerce(kind: Kind, v: unknown, column: string): unknown {
const element = kind.slice(0, -2);
if (typeof v === "string") return parseArray(v, (s) => coerce(element, s, column));
if (Array.isArray(v)) return v.map((e) => (Array.isArray(e) ? coerce(kind, e, column) : coerce(element, e, column)));
// A MySQL VECTOR's bytes: no driver gives any other array of floats as bytes.
if (element === "float" && isBytes(v)) return float32s(v, column);
// Bun.sql's int4[] and float4[] are typed arrays.
if (ArrayBuffer.isView(v) && !(v instanceof DataView)) return Array.from(v as unknown as ArrayLike<unknown>, (e) => coerce(element, e, column));
return v;
Expand Down
45 changes: 45 additions & 0 deletions tests/values.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -363,3 +363,48 @@ test("MySQL: a truth value is a boolean on every driver, whatever form it gives
assert.deepEqual(await execute(bun(1) as never, q), expected);
await assert.rejects(execute(bun(5) as never, q) as Promise<unknown>, /Column "done" is a boolean, stored as 0 or 1, and holds 5/);
});

test("MySQL: a VECTOR is an array of its floats on every driver that reads one, and its parameter binds their text", async () => {
const vector: EngineType = { kind: "array", of: scalar("float4"), nullable_items: false };
assert.equal(columnKind(vector, "mysql"), "float[]");
assert.equal(renderType(vector), "number[]");
assert.equal(renderParamType(vector, "mysql"), "number[]");
// The server sends a VECTOR as its floats' bytes, 32-bit and little-endian (MySQL 9.7.2).
const floats = [1, 2.5, -3, 0.10000000149011612];
const bytes = Buffer.alloc(16);
floats.forEach((f, i) => bytes.writeFloatLE(f, i * 4));
const columns: Columns = [["v", "float[]"], ["vn", "float[]"]];
const q: QueryModule<"mysql"> = { dialect: "mysql", sql: "select 1", params: [], columns };
const expected = [{ v: floats, vn: null }];
// mysql2 3.x decodes the floats itself; mariadb and PlanetScale give the bytes.
const fields = [{ name: "v", columnType: 242, decimals: 0 }, { name: "vn", columnType: 242, decimals: 0 }];
assert.deepEqual(await execute({ query: async () => [[], []], execute: async () => [[{ v: floats, vn: null }], fields] } as never, q), expected);
assert.deepEqual(await execute({ query: async () => [[], []], execute: async () => [[{ v: Buffer.from(bytes), vn: null }], fields] } as never, q), expected);
assert.deepEqual([...((await execute({ batch: async () => ({}), query: async () => [{ v: Buffer.from(bytes), vn: null }] } as never, q)) as unknown[])], expected);
const planetscale = {
execute: async (_: string, __: unknown, options: { cast: (f: { type: string; charset?: number }, v: string | null) => unknown }) => ({
rows: [{ v: options.cast({ type: "VECTOR", charset: 63 }, String.fromCharCode(...bytes)), vn: options.cast({ type: "VECTOR", charset: 63 }, null) }],
}),
};
assert.deepEqual(await execute(planetscale as never, q), expected);
// A slice of a larger buffer, as a driver's row may be, reads from its own offset.
assert.deepEqual(coerce("float[]", Buffer.concat([Buffer.of(9, 9, 9), bytes]).subarray(3), "v"), floats);
assert.throws(() => coerce("float[]", bytes.subarray(0, 5), "v"), (e: Error) => e instanceof DecodeError && /Column "v" is a VECTOR, whose value is 4 bytes per 32-bit float, and holds 5 bytes/.test(e.message));
// A Postgres float4[] from Bun.sql is a Float32Array, which is not read as bytes.
assert.deepEqual(coerce("float[]", new Float32Array([1, 2.5]), "v"), [1, 2.5]);

// A parameter binds the text of its floats, which the engine's `to_vector(?)` reads, on every driver.
const w: QueryModule<"mysql"> = { dialect: "mysql", sql: "insert into items (v) values (to_vector(:v))", params: ["v"], inputs: { v: { type: vector } } };
const seen: unknown[][] = [];
await execute({ query: async () => [[], []] as [unknown, unknown], execute: async (_: unknown, v?: unknown[]) => (seen.push(v!), [[], []] as [unknown, unknown]) } as never, w, { v: floats });
await execute({ query: async (_: string, v: unknown) => (seen.push(v as unknown[]), []), batch: async () => [] } as never, w, { v: floats });
await execute({ unsafe: async (_: string, v?: unknown[]) => (seen.push(v!), []), commitDistributed: async () => {} } as never, w, { v: floats });
await execute({ execute: async (_: string, v: unknown[]) => (seen.push(v), { rows: [] }) } as never, w, { v: floats });
await execute({ batch: async () => ({}), query: async (_: string, v: unknown) => (seen.push(v as unknown[]), []) } as never, w, { v: null });
assert.deepEqual(seen, [...Array(4).fill(["[1,2.5,-3,0.10000000149011612]"]), [null]]);
const fake = { query: async () => [[], []], execute: async () => [[], []] } as never;
await assert.rejects(execute(fake, w, { v: [1, "2"] }) as Promise<unknown>, /^TypeError: Parameter "v" is a VECTOR, an array of finite numbers, and the element at \[1\] is "2"\.$/);
await assert.rejects(execute(fake, w, { v: [1, NaN] }) as Promise<unknown>, /the element at \[1\] is NaN/);
await assert.rejects(execute(fake, w, { v: [Infinity] }) as Promise<unknown>, /the element at \[0\] is Infinity/);
await assert.rejects(execute(fake, w, { v: [1, null] }) as Promise<unknown>, /^TypeError: Parameter "v" is an array whose elements may not be null, and the element at \[1\] is/);
});
Loading
0