8000 Implement lexer by rpiaggio · Pull Request #861 · gemini-hlsw/clue · GitHub
[go: up one dir, main page]
Skip to content

Implement lexer #861

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rpiaggio wants to merge 1 commit into from
Closed

Implement lexer #861

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 179 additions & 0 deletions core/src/main/scala/clue/GraphQLDocumentScan.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
// Copyright (c) 2016-2026 Association of Universities for Research in Astronomy, Inc. (AURA)
// For license information see LICENSE or https://opensource.org/licenses/BSD-3-Clause

package clue

/**
* Token-level utilities over `Vector[GraphQLLexer.Token]`, for the `gql` macro. Macro-free so it
* can be unit-tested.
*/
private[clue] object GraphQLDocumentScan {
import GraphQLLexer.Token

/** A GraphQL type reference as written in a var-def. */
sealed trait VarType { def nonNull: Boolean; def render: String }
object VarType {
final case class Named(name: String, nonNull: Boolean) extends VarType {
def render: String = if (nonNull) s"$name!" else name
}
final case class ListOf(inner: VarType, nonNull: Boolean) extends VarType {
def render: String = s"[${inner.render}]" + (if (nonNull) "!" else "")
}
}
import VarType.*

private def stripTopNonNull(t: VarType): VarType = t match {
case Named(n, _) => Named(n, nonNull = false)
case ListOf(i, _) => ListOf(i, nonNull = false)
}

/**
* GraphQL "is variable usage allowed", as `main` implements it today: same type modulo top-level
* nullability, and a non-null requirement needs a non-null declaration.
*/
def usableAs(declared: VarType, required: VarType): Boolean =
stripTopNonNull(declared) == stripTopNonNull(
required
) && (!required.nonNull || declared.nonNull)

final case class ScanError(message: String, pos: Int)

private def posOf(tokens: Vector[Token], i: Int): Int = tokens.lift(i).map(_.pos).getOrElse(-1)

private def isPunct(tokens: Vector[Token], i: Int, v: String): Boolean =
tokens.lift(i) match {
case Some(Token.Punct(p, _)) => p == v
case _ => false
}

// Type :: NamedType "!"? | ListType "!"?
private def parseType(tokens: Vector[Token], i: Int): Either[ScanError, (VarType, Int)] =
tokens.lift(i) match {
case Some(Token.Name(n, _)) =>
val nn = isPunct(tokens, i + 1, "!")
Right((Named(n, nn), i + (if (nn) 2 else 1)))
case Some(Token.Punct("[", _)) =>
parseType(tokens, i + 1).flatMap { case (inner, j) =>
if (!isPunct(tokens, j, "]")) Left(ScanError("expected ']'", posOf(tokens, j)))
else {
val nn = isPunct(tokens, j + 1, "!")
Right((ListOf(inner, nn), j + (if (nn) 2 else 1)))
}
}
case _ => Left(ScanError("expected a type", posOf(tokens, i)))
}

// Skip one Value: a single scalar/enum token, a `$variable`, or a balanced `[...]`/`{...}` (nested
// strings are already atomic tokens, so counting only the outermost bracket kind is enough).
private def skipValue(tokens: Vector[Token], i: Int): Either[ScanError, Int] =
tokens.lift(i) match {
case Some(Token.Punct(o @ ("[" | "{" | "("), _)) =>
val c = o match { case "[" => "]"; case "{" => "}"; case _ => ")" }
def loop(j: Int, depth: Int): Either[ScanError, Int] =
tokens.lift(j) match {
case Some(Token.Punct(`o`, _)) => loop(j + 1, depth + 1)
case Some(Token.Punct(`c`, _)) =>
if (depth == 1) Right(j + 1) else loop(j + 1, depth - 1)
case Some(_) => loop(j + 1, depth)
case None => Left(ScanError(s"unterminated '$o'", posOf(tokens, i)))
}
loop(i + 1, 1)
case Some(Token.Punct("$", _)) =>
tokens.lift(i + 1) match {
case Some(Token.Name(_, _)) => Right(i + 2)
case _ => Left(ScanError("expected variable name", posOf(tokens, i + 1)))
}
case Some(_) => Right(i + 1)
case None => Left(ScanError("expected a value", posOf(tokens, i)))
}

// Skip an optional `= Value` default, then any number of `@ Name (Arguments)?` directives.
private def skipDefaultAndDirectives(tokens: Vector[Token], i: Int): Either[ScanError, Int] = {
def directives(j: Int): Either[ScanError, Int] =
if (!isPunct(tokens, j, "@")) Right(j)
else
tokens.lift(j + 1) match {
case Some(Token.Name(_, _)) =>
val afterName = j + 2
(if (isPunct(tokens, afterName, "(")) skipValue(tokens, afterName)
else Right(afterName))
.flatMap(directives)
case _ => Left(ScanError("expected directive name", posOf(tokens, j + 1)))
}

(if (isPunct(tokens, i, "=")) skipValue(tokens, i + 1) else Right(i)).flatMap(directives)
}

/**
* Parse a parenthesized var-def list `($a: T = default @dir, $b: [U!]!)` given as tokens (the
* whole token stream is the list, parens included). Defaults (any Value, possibly nested
* lists/objects/strings) and directives are skipped.
*/
def parseVarDefs(tokens: Vector[Token]): Either[ScanError, Map[String, VarType]] = {
def loop(i: Int, acc: Map[String, VarType]): Either[ScanError, Map[String, VarType]] =
if (isPunct(tokens, i, ")")) Right(acc)
else if (!isPunct(tokens, i, "$")) Left(ScanError("expected '$' or ')'", posOf(tokens, i)))
else
tokens.lift(i + 1) match {
case Some(Token.Name(name, _)) =>
if (!isPunct(tokens, i + 2, ":")) Left(ScanError("expected ':'", posOf(tokens, i + 2)))
else
parseType(tokens, i + 3).flatMap { case (tpe, j) =>
skipDefaultAndDirectives(tokens, j).flatMap(next => loop(next, acc + (name -> tpe)))
}
case _ => Left(ScanError("expected variable name", posOf(tokens, i + 1)))
}

if (!isPunct(tokens, 0, "(")) Left(ScanError("expected '('", posOf(tokens, 0)))
else loop(1, Map.empty)
}

/**
* The operation header's var-defs of a document: skip to the first
* `query`/`mutation`/`subscription` Name at brace-depth 0 (fragments may precede it), skip an
* optional operation Name, and if the next token is `(` parse the var-defs up to the matching
* `)`. A shorthand `{ ... }` document, or an operation without parens, declares none. Returns the
* map and the index of the first token AFTER the header (or after the keyword/name when there is
* no header; 0 when no keyword is found).
*/
def operationVars(tokens: Vector[Token]): Either[ScanError, (Map[String, VarType], Int)] = {
@scala.annotation.tailrec
def findHeader(i: Int, depth: Int): Int =
if (i >= tokens.length) -1
else
tokens(i) match {
case Token.Punct("{", _) => findHeader(i + 1, depth + 1)
case Token.Punct("}", _) => findHeader(i + 1, depth - 1)
case Token.Name(n, _)
if depth == 0 && (n == "query" || n == "mutation" || n == "subscription") =>
i
case _ => findHeader(i + 1, depth)
}

@scala.annotation.tailrec
def matchParen(j: Int, depth: Int): Int =
if (j >= tokens.length) -1
else
tokens(j) match {
case Token.Punct("(", _) => matchParen(j + 1, depth + 1)
case Token.Punct(")", _) => if (depth == 1) j else matchParen(j + 1, depth - 1)
case _ => matchParen(j + 1, depth)
}

val kwIdx = findHeader(0, 0)
if (kwIdx < 0) Right((Map.empty, 0))
else {
val afterKw = kwIdx + 1
val afterName = tokens.lift(afterKw) match {
case Some(Token.Name(_, _)) => afterKw + 1
case _ => afterKw
}
if (!isPunct(tokens, afterName, "(")) Right((Map.empty, afterName))
else {
val closeIdx = matchParen(afterName + 1, 1)
if (closeIdx < 0) Left(ScanError("unterminated var-defs", posOf(tokens, afterName)))
else parseVarDefs(tokens.slice(afterName, closeIdx + 1)).map(vars => (vars, closeIdx + 1))
}
}
}
}
118 changes: 50 additions & 68 deletions core/src/main/scala/clue/GraphQLInterpolator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,15 @@ extension (inline sc: StringContext)
* Builds an operation `document` or a subquery's `subquery`, splicing subqueries inline. At
* runtime it produces exactly the string the standard `s"..."` interpolator would.
*
* At compile time it runs the *caller-check*: for every spliced value that declares variables (a
* `GraphQLSubquery` with a `type VariableDefs` member), it verifies that the variables are
* declared where the splice happens, with a compatible ("usable as") type. The declaration is
* read from the operation's `query (...)` header, or — when the splice happens inside a
* `GraphQLSubquery` — from that subquery's own `type VariableDefs`, so nesting is checked one
* At compile time the assembled document (all literal parts, with spliced holes) is lexed per the
* GraphQL spec — a lexical error (an unterminated string, an unknown character, ...) is a compile
* error — and the *caller-check* runs over the resulting tokens: for every spliced value that
* declares variables (a `GraphQLSubquery` with a `type VariableDefs` member), it verifies that
* the variables are declared where the splice happens, with a compatible ("usable as") type. The
* declaration is re 10BC8 ad from the operation's `query (...)` header — found by scanning tokens for
* the `query`/`mutation`/`subscription` keyword at brace-depth 0, so fragment definitions may
* precede the operation without its header being missed — or, when the splice happens inside a
* `GraphQLSubquery`, from that subquery's own `type VariableDefs`, so nesting is checked one
* level at a time and holds transitively.
*
* Requirements are read straight off the spliced subquery's type, so the check also works when
Expand Down Expand Up @@ -112,95 +116,73 @@ private[clue] object GraphQLInterpolator {
loop(Symbol.spliceOwner)
}

def splitTopLevel(s: String): List[String] = {
val out = scala.collection.mutable.ListBuffer.empty[String]
val cur = new StringBuilder
var depth = 0
s.foreach {
case '[' => depth += 1; cur += '['
case ']' => depth -= 1; cur += ']'
case ',' if depth == 0 => out += cur.toString; cur.clear()
case c => cur += c
// Parse a `VariableDefs` literal (`"($a: T, ...)"`, hand-written on a `GraphQLSubquery`, unlike
// the document itself) into its declared types, aborting with a compile error naming `ownerName`
// if it doesn't lex/parse.
def parseVariableDefs(
s: String,
ownerName: String
): Map[String, GraphQLDocumentScan.VarType] = {
val parsed = for {
tokens <- GraphQLLexer.tokenize(s).left.map(_.message)
vars <- GraphQLDocumentScan.parseVarDefs(tokens).left.map(_.message)
} yield vars
parsed match {
case Right(vars) => vars
case Left(msg) =>
report.errorAndAbort(s"gql: invalid VariableDefs [$s] on $ownerName: $msg")
}
if (cur.nonEmpty) out += cur.toString
out.toList
}

// Parse a parenthesized var-def list `($a: T, $b: U)` into name -> GraphQL type.
def parseVarDefs(s0: String): Map[String, String] = {
val s = s0.trim.stripPrefix("(").stripSuffix(")").trim.replace("$$", "$")
if (s.isEmpty) Map.empty
else
splitTopLevel(s).flatMap { entry =>
val e = entry.trim
val nameEnd = e.indexOf(':')
if (nameEnd < 0 || !e.startsWith("$")) None
else {
val name = e.substring(1, nameEnd).trim
val tpe = e.substring(nameEnd + 1).takeWhile(_ != '=').trim
Some(name -> tpe)
}
}.toMap
}

// Extract and parse the operation header `(...)` from `query (...) ...` (the first literal part).
def operationVars(header: String): Map[String, String] = {
val h = header.replace("$$", "$")
val open = h.indexOf('(')
if (open < 0) Map.empty
else {
var depth = 0; var i = open; var end = -1
while (i < h.length && end < 0) {
h.charAt(i) match {
case '(' => depth += 1
case ')' => depth -= 1; if (depth == 0) end = i
case _ => ()
}
i += 1
}
if (end < 0) Map.empty else parseVarDefs(h.substring(open, end + 1))
// Lex the whole document (literal parts + splice holes) and find the operation header's
// var-defs. Scanning tokens (rather than the old `indexOf('(')` on raw strings) means a fragment
// preceding the operation, or a string/comment containing `(`, can't be mistaken for the header.
val tokens: Vector[GraphQLLexer.Token] =
GraphQLLexer.tokenizeParts(parts) match {
case Right(ts) => ts
case Left(err) => report.errorAndAbort(s"gql: ${err.message}", Position.ofMacroExpansion)
}
}

// GraphQL "is variable usage allowed": the declared type must be usable where the required type
// is expected. Same base type, and a non-null requirement needs a non-null declared type.
def usableAs(declaredType: String, reqType: String): Boolean = {
val d = declaredType.trim; val r = reqType.trim
d.stripSuffix("!").trim == r.stripSuffix("!").trim && (!r.endsWith("!") || d.endsWith("!"))
}
val headerVars: Map[String, GraphQLDocumentScan.VarType] =
GraphQLDocumentScan.operationVars(tokens) match {
case Right((vars, _)) => vars
case Left(err) => report.errorAndAbort(s"gql: ${err.message}", Position.ofMacroExpansion)
}

// What the splice site declares. An operation declares its variables in the document header; a
// subquery declares them in `type VariableDefs`. Both are read (they are never both populated in
// practice), so a splice is checked against everything in scope where it happens.
val enclosingVariableDefs: Map[String, String] =
val enclosingVariableDefs: Map[String, GraphQLDocumentScan.VarType] =
enclosingSubquery
.flatMap(cls => variableDefsOf(cls.typeRef))
.map(parseVarDefs)
.flatMap(cls =>
variableDefsOf(cls.typeRef).map(parseVariableDefs(_, cls.name.stripSuffix("$")))
)
.getOrElse(Map.empty)

val declaredVars: Map[String, String] =
enclosingVariableDefs ++ parts.headOption.map(operationVars).getOrElse(Map.empty)
val declaredVars: Map[String, GraphQLDocumentScan.VarType] = enclosingVariableDefs ++ headerVars

// Where the missing declaration has to be added. `name` keeps the module-class `$` suffix for an
// `object` subquery, which would only confuse the reader.
val declarationSite: String =
enclosingSubquery.fold("operation")(cls => s"subquery [${cls.name.stripSuffix("$")}]")

argExprs.foreach { arg =>
variableDefsOf(arg.asTerm.tpe).foreach { required =>
parseVarDefs(required).foreach { case (name, reqType) =>
val argTpe = arg.asTerm.tpe
variableDefsOf(argTpe).foreach { requiredDefs =>
val required = parseVariableDefs(requiredDefs, argTpe.typeSymbol.name.stripSuffix("$"))
required.foreach { case (name, reqType) =>
declaredVars.get(name) match {
case None =>
case None =>
report.errorAndAbort(
s"gql: $declarationSite does not declare variable $$$name required by a spliced subquery (needs $reqType)",
s"gql: $declarationSite does not declare variable $$$name required by a spliced subquery (needs ${reqType.render})",
arg.asTerm.pos
)
case Some(declared) if !usableAs(declared, reqType) =>
case Some(declared) if !GraphQLDocumentScan.usableAs(declared, reqType) =>
report.errorAndAbort(
s"gql: $declarationSite declares $$$name: $declared but a spliced subquery requires it usable as $reqType",
s"gql: $declarationSite declares $$$name: ${declared.render} but a spliced subquery requires it usable as ${reqType.render}",
arg.asTerm.pos
)
case _ => ()
case _ => ()
}
}
}
Expand Down
Loading
0