8000 feat: queryMemoryLimit and QRY-012 (minigraf#549) by adityamukho · Pull Request #16 · project-minigraf/minigraf-java · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ path = "src/uniffi_bindgen.rs"
[dependencies]
# Pin to the exact released version. The release workflow updates this line.
# Unreleased v3 core (format v8). Switches to minigraf = "=3.0.0" at the v3.0.0 release.
minigraf = { git = "https://github.com/project-minigraf/minigraf", rev = "97cd4a2a85301ea3478250dd654139d45a1f7cd0" }
minigraf = { git = "https://github.com/project-minigraf/minigraf", rev = "df7282f1f3ad17628d33fa5488730201313ac268" }
uniffi = { version = "0.31.1", features = ["cli"] }
thiserror = "2.0.18"
serde_json = "1.0.149"
Expand Down
6 changes: 6 additions & 0 deletions java/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,12 @@ try (MiniGrafFactLog log = src.factLog(Minigraf.filter().build());
} // close() without finish() abandons the build and leaves no file
```

`Minigraf.options().queryMemoryLimit(bytes)` caps the bytes one query may hold, counted
per entry by the operators that keep state (join build sides, negation and `or` tables,
distinct sets, aggregate groups, window rows, rule-derived facts). A query that would
hold more fails with QRY-012 and frees what it held. The default, and `Long.MAX_VALUE`,
is no limit. Rows a cursor returns are not counted, so read a large answer with `query()`.

Kotlin callers can use `cursor.batches()` and `log.records()` directly, and the
generated constructors with named arguments (`OpenOptions(readOnly = true)`).

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,14 @@ class OpenOptionsBuilder internal constructor() {
fun allowUnlocked(value: Boolean) = apply { options = options.copy(allowUnlocked = value) }
fun walCheckpointThreshold(entries: Long) = apply { options = options.copy(walCheckpointThreshold = entries) }
fun maxDerivedFacts(n: Long) = apply { options = options.copy(maxDerivedFacts = n) }
/** Deprecated alias of [maxDerivedFacts]; the lower of the two applies. */
fun maxResults(n: Long) = apply { options = options.copy(maxResults = n) }
fun synchronous(mode: SyncMode) = apply { options = options.copy(synchronous = mode) }
/**
* The most bytes one query may hold; a query that would hold more fails with
* QRY-012. `Long.MAX_VALUE` means no limit, the default.
*/
fun queryMemoryLimit(bytes: Long) = apply { options = options.copy(queryMemoryLimit = bytes) }
fun build(): OpenOptions = options
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,21 @@ class EtlTest {
assertTrue(File("$p.wal").exists(), "WAL kept on close (#322)")
}

@Test
fun queryMemoryLimitIsQry012() {
// A cartesian product holds one side in full.
val product = "(query [:find ?a ?b :where [?a :n _] [?b :m _]])"
Minigraf.openWithOptions(path("small.graph"), Minigraf.options().queryMemoryLimit(64).build()).use {
it.execute("(transact [[:a :n 1] [:b :m 2]])")
val e = assertThrows<MiniGrafException.Query> { it.execute(product) }
assertTrue(code(e).startsWith("[QRY-012]"))
}
Minigraf.openWithOptions(path("max.graph"), Minigraf.options().queryMemoryLimit(Long.MAX_VALUE).build()).use {
it.execute("(transact [[:a :n 1] [:b :m 2]])")
it.execute(product)
}
}

@Test
fun factLogRecordsAndFilters() {
val db = source(path("s.graph"))
Expand Down
60 changes: 56 additions & 4 deletions 60 src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,12 +126,17 @@ pub struct OpenOptions {
/// Facts a recursive rule may derive per iteration. Default 1,000,000.
#[uniffi(default)]
pub max_derived_facts: Option<i64>,
/// Rows a query may return. Default 1,000,000.
/// Deprecated alias of `max_derived_facts`; the lower of the two applies.
#[uniffi(default)]
pub max_results: Option<i64>,
/// WAL durability. Default `Full`.
#[uniffi(default)]
pub synchronous: Option<SyncMode>,
/// The most bytes one query may hold; a query that would hold more fails
/// with QRY-012. The maximum (i64::MAX, 9223372036854775807) means no
/// limit, the default.
#[uniffi(default)]
pub query_memory_limit: Option<i64>,
}

impl OpenOptions {
Expand Down Expand Up @@ -160,14 +165,27 @@ impl OpenOptions {
o = o.max_derived_facts(to_usize(v, "max_derived_facts")?);
}
if let Some(v) = self.max_results {
o = o.max_results(to_usize(v, "max_results")?);
#[allow(deprecated)]
{
o = o.max_results(to_usize(v, "max_results")?);
}
}
if let Some(v) = self.synchronous {
o = o.synchronous(match v {
SyncMode::Full => minigraf::SyncMode::Full,
SyncMode::Normal => minigraf::SyncMode::Normal,
});
}
if let Some(v) = self.query_memory_limit {
// i64::MAX stands for core's usize::MAX (no limit), as for
// wal_checkpoint_threshold.
let n = if v == i64::MAX {
usize::MAX
} else {
to_usize(v, "query_memory_limit")?
};
o = o.query_memory_limit(n);
}
Ok(o)
}
}
Expand Down Expand Up @@ -717,7 +735,7 @@ mod tests {

#[test]
fn value_to_json_ref() {
let id = minigraf::EntityId::new_v4();
let id = minigraf::EntityId::from_u128(0x6f1c_2a3b_4d5e_4f60_8172_839a_b4c5_d6e7);
let j = value_to_json(&Value::Ref(id));
assert_eq!(j, serde_json::Value::String(id.to_string()));
}
Expand Down Expand Up @@ -832,7 +850,7 @@ mod tests {

#[test]
fn value_round_trips_through_core() {
let id = minigraf::EntityId::new_v4();
let id = minigraf::EntityId::from_u128(0x6f1c_2a3b_4d5e_4f60_8172_839a_b4c5_d6e7);
for v in [
Value::String("s".into()),
Value::Integer(-3),
Expand Down Expand Up @@ -914,6 +932,40 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn query_memory_limit_is_qry_012() {
let options = |limit| OpenOptions {
query_memory_limit: Some(limit),
..OpenOptions::default()
};
assert!(msg(err(options(-1).to_core())).starts_with("[API-017]"));

let dir = std::env::temp_dir().join(format!("minigraf_ffi_qml_{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).expect("dir");
// A cartesian product holds one side in full.
let product = "(query [:find ?a ?b :where [?a :n _] [?b :m _]])";
for (name, limit, fails) in [("small", 64, true), ("max", i64::MAX, false)] {
let path = dir.join(format!("{name}.graph"));
let db =
MiniGrafDb::open_with_options(path.to_str().expect("utf8").into(), options(limit))
.expect("open");
db.execute("(transact [[:a :n 1] [:b :m 2]])".into())
.expect("transact");
let r = db.execute(product.into());
if fails {
let e = err(r);
assert!(matches!(e, MiniGrafError::Query { .. }), "a query error");
assert!(msg(e).starts_with("[QRY-012]"));
let cursor = db.query(product.into()).expect("query");
assert!(msg(err(cursor.next_batch(10))).starts_with("[QRY-012]"));
} else {
r.expect("no limit");
}
}
let _ = std::fs::remove_dir_all(&dir);
}

#[test]
fn cursor_batches_and_closes() {
let db = MiniGrafDb::open_in_memory().expect("open");
Expand Down
Loading
0