From a2898fcda78001fc9ab3748be83984ae52b36eae Mon Sep 17 00:00:00 2001 From: Edoardo Luppi Date: Wed, 31 Jan 2024 12:11:33 +0100 Subject: [PATCH 1/3] refactor: fix nullability for TokenStream.getText parameters --- .../org/antlr/v4/kotlinruntime/BufferedTokenStream.kt | 9 ++++++--- .../org/antlr/v4/kotlinruntime/CommonTokenStream.kt | 1 - .../org/antlr/v4/kotlinruntime/DummyTokenStream.kt | 2 +- .../kotlin/org/antlr/v4/kotlinruntime/TokenStream.kt | 3 +-- .../org/antlr/v4/kotlinruntime/UnbufferedTokenStream.kt | 9 ++++++--- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/BufferedTokenStream.kt b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/BufferedTokenStream.kt index 7bb134da..50aa46b3 100644 --- a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/BufferedTokenStream.kt +++ b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/BufferedTokenStream.kt @@ -1,6 +1,5 @@ // Copyright 2017-present Strumenta and contributors, licensed under Apache 2.0. // Copyright 2024-present Strumenta and contributors, licensed under BSD 3-Clause. - package org.antlr.v4.kotlinruntime import com.strumenta.antlrkotlin.runtime.assert @@ -474,8 +473,12 @@ public open class BufferedTokenStream(tokenSource: TokenSource) : TokenStream { override fun getText(ctx: RuleContext): String = getText(ctx.sourceInterval) - override fun getText(start: Token, stop: Token): String? = - getText(Interval.of(start.tokenIndex, stop.tokenIndex)) + override fun getText(start: Token?, stop: Token?): String? = + if (start != null && stop != null) { + getText(Interval.of(start.tokenIndex, stop.tokenIndex)) + } else { + "" + } /** * Get all tokens from lexer until `EOF`. diff --git a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/CommonTokenStream.kt b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/CommonTokenStream.kt index a40543e6..0b58136f 100644 --- a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/CommonTokenStream.kt +++ b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/CommonTokenStream.kt @@ -1,6 +1,5 @@ // Copyright 2017-present Strumenta and contributors, licensed under Apache 2.0. // Copyright 2024-present Strumenta and contributors, licensed under BSD 3-Clause. - package org.antlr.v4.kotlinruntime /** diff --git a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/DummyTokenStream.kt b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/DummyTokenStream.kt index f1085ca7..5ea8d4e1 100644 --- a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/DummyTokenStream.kt +++ b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/DummyTokenStream.kt @@ -53,6 +53,6 @@ internal object DummyTokenStream : TokenStream { override fun getText(ctx: RuleContext): String = throw UnsupportedOperationException() - override fun getText(start: Token, stop: Token): String = + override fun getText(start: Token?, stop: Token?): String = throw UnsupportedOperationException() } diff --git a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/TokenStream.kt b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/TokenStream.kt index b79e11e9..f5c2e576 100644 --- a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/TokenStream.kt +++ b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/TokenStream.kt @@ -1,6 +1,5 @@ // Copyright 2017-present Strumenta and contributors, licensed under Apache 2.0. // Copyright 2024-present Strumenta and contributors, licensed under BSD 3-Clause. - package org.antlr.v4.kotlinruntime import org.antlr.v4.kotlinruntime.misc.Interval @@ -134,5 +133,5 @@ public interface TokenStream : IntStream { * @throws UnsupportedOperationException If this stream does not support * this method for the specified tokens */ - public fun getText(start: Token, stop: Token): String? + public fun getText(start: Token?, stop: Token?): String? } diff --git a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/UnbufferedTokenStream.kt b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/UnbufferedTokenStream.kt index 11bdbf4b..2ea16ed0 100644 --- a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/UnbufferedTokenStream.kt +++ b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/UnbufferedTokenStream.kt @@ -1,6 +1,5 @@ // Copyright 2017-present Strumenta and contributors, licensed under Apache 2.0. // Copyright 2024-present Strumenta and contributors, licensed under BSD 3-Clause. - package org.antlr.v4.kotlinruntime import com.strumenta.antlrkotlin.runtime.System @@ -112,8 +111,12 @@ public open class UnbufferedTokenStream(override val tokenSource: TokenSource, b override fun getText(ctx: RuleContext): String = getText(ctx.sourceInterval) - override fun getText(start: Token, stop: Token): String = - getText(Interval.of(start.tokenIndex, stop.tokenIndex)) + override fun getText(start: Token?, stop: Token?): String = + if (start != null && stop != null) { + getText(Interval.of(start.tokenIndex, stop.tokenIndex)) + } else { + "" + } override fun consume() { if (LA(1) == Token.EOF) { From e77b7ecdcd62d9de94e1cb0942268f09fc58779f Mon Sep 17 00:00:00 2001 From: Edoardo Luppi Date: Wed, 31 Jan 2024 12:12:27 +0100 Subject: [PATCH 2/3] refactor: remove unnecessary overload for ATNDeserializer.decodeIntsEncodedAs16BitWords --- .../antlr/v4/kotlinruntime/atn/ATNDeserializer.kt | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/atn/ATNDeserializer.kt b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/atn/ATNDeserializer.kt index 24222efb..9a864d71 100644 --- a/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/atn/ATNDeserializer.kt +++ b/antlr-kotlin-runtime/src/commonMain/kotlin/org/antlr/v4/kotlinruntime/atn/ATNDeserializer.kt @@ -174,16 +174,14 @@ public open class ATNDeserializer(deserializationOptions: ATNDeserializationOpti for (state in atn.states) { if (state is BlockStartState) { // We need to know the end state to set its start state - if (state.endState == null) { - throw IllegalStateException() - } + val endState = state.endState ?: throw IllegalStateException() // Block end states can only be associated to a single block start state - if (state.endState!!.startState != null) { + if (endState.startState != null) { throw IllegalStateException() } - state.endState!!.startState = state + endState.startState = state } if (state is PlusLoopbackState) { @@ -538,13 +536,10 @@ public open class ATNDeserializer(deserializationOptions: ATNDeserializationOpti LexerActionType.TYPE -> LexerTypeAction(data1) } - public open fun decodeIntsEncodedAs16BitWords(data16: CharArray): IntArray = - decodeIntsEncodedAs16BitWords(data16, false) - /** * Convert a list of chars (16 uint) that represent a serialized and compressed list of ints for an ATN. */ - public open fun decodeIntsEncodedAs16BitWords(data16: CharArray, trimToSize: Boolean): IntArray { + public open fun decodeIntsEncodedAs16BitWords(data16: CharArray, trimToSize: Boolean = false): IntArray { // Will be strictly smaller, but we waste bit of space to avoid copying during initialization of parsers val data = IntArray(data16.size) var i = 0 From f2f975414333da8764b2b799897cf0d352163a80 Mon Sep 17 00:00:00 2001 From: Edoardo Luppi Date: Wed, 31 Jan 2024 12:14:14 +0100 Subject: [PATCH 3/3] refactor: fix code generation --- .../antlr/v4/codegen/target/KotlinTarget.kt | 9 +--- .../tool/templates/codegen/Kotlin/Kotlin.stg | 53 ++++++++++--------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/antlr-kotlin-target/src/main/kotlin/org/antlr/v4/codegen/target/KotlinTarget.kt b/antlr-kotlin-target/src/main/kotlin/org/antlr/v4/codegen/target/KotlinTarget.kt index 6e77032a..37695b0f 100644 --- a/antlr-kotlin-target/src/main/kotlin/org/antlr/v4/codegen/target/KotlinTarget.kt +++ b/antlr-kotlin-target/src/main/kotlin/org/antlr/v4/codegen/target/KotlinTarget.kt @@ -5,7 +5,6 @@ package org.antlr.v4.codegen.target import org.antlr.v4.codegen.CodeGenerator import org.antlr.v4.codegen.Target import org.antlr.v4.codegen.UnicodeEscapes -import org.antlr.v4.tool.Grammar public class KotlinTarget(codeGenerator: CodeGenerator) : JavaTarget(codeGenerator) { private companion object { @@ -70,6 +69,7 @@ public class KotlinTarget(codeGenerator: CodeGenerator) : JavaTarget(codeGenerat addAll(kotlinKeywords) add("rule") add("parserRule") + add("ruleIndex") } override fun getVersion(): String = @@ -92,13 +92,6 @@ public class KotlinTarget(codeGenerator: CodeGenerator) : JavaTarget(codeGenerat override fun appendUnicodeEscapedCodePoint(codePoint: Int, sb: StringBuilder): Unit = UnicodeEscapes.appendEscapedCodePoint(sb, codePoint, "Java") - override fun getTokenTypeAsTargetLabel(g: Grammar, ttype: Int): String { - // All tokens are namespaced inside a Tokens object. - // Here we simply force the qualification - val label = super.getTokenTypeAsTargetLabel(g, ttype) - return "Tokens.$label" - } - override fun getTargetStringLiteralFromANTLRStringLiteral( generator: CodeGenerator, literal: String, diff --git a/antlr-kotlin-target/src/main/resources/org/antlr/v4/tool/templates/codegen/Kotlin/Kotlin.stg b/antlr-kotlin-target/src/main/resources/org/antlr/v4/tool/templates/codegen/Kotlin/Kotlin.stg index 38ae179a..cdbc5c22 100644 --- a/antlr-kotlin-target/src/main/resources/org/antlr/v4/tool/templates/codegen/Kotlin/Kotlin.stg +++ b/antlr-kotlin-target/src/main/resources/org/antlr/v4/tool/templates/codegen/Kotlin/Kotlin.stg @@ -57,6 +57,7 @@ import org.antlr.v4.kotlinruntime.atn.ATN.Companion.INVALID_ALT_NUMBER import org.antlr.v4.kotlinruntime.dfa.* import org.antlr.v4.kotlinruntime.misc.* import org.antlr.v4.kotlinruntime.tree.* +import kotlin.jvm.JvmField >> @@ -293,7 +294,7 @@ public open class (input: TokenStream) : -> return _sempred(_localctx as , predIndex)}; separator="\n"> + -> return _sempred(_localctx as ?, predIndex)}; separator="\n"> } return true @@ -322,7 +323,7 @@ dumpActions(recog, argFuncs, actionFuncs, sempredFuncs) ::= << override fun action(_localctx: RuleContext?, ruleIndex: Int, actionIndex: Int) { when (ruleIndex) { - -> _action(_localctx as , actionIndex)}; separator="\n"> + -> _action(_localctx, actionIndex)}; separator="\n"> } } @@ -332,7 +333,7 @@ override fun action(_localctx: RuleContext?, ruleIndex: Int, actionIndex: Int) { override fun sempred(_localctx: RuleContext?, ruleIndex: Int, predIndex: Int): Boolean { when (ruleIndex) { - -> return _sempred(_localctx as , predIndex)}; separator="\n"> + -> return _sempred(_localctx, predIndex)}; separator="\n"> } return true @@ -346,7 +347,7 @@ override fun sempred(_localctx: RuleContext?, ruleIndex: Int, predIndex: Int): B * overriding implementation impossible to maintain. */ RuleActionFunction(r, actions) ::= << -private fun _action(_localctx: , actionIndex: Int) { +private fun _action(_localctx: ?, actionIndex: Int) { when (actionIndex) { -> { @@ -356,11 +357,15 @@ private fun _action(_localctx: , actionIndex: Int) { } >> -/* This generates a private method since the predIndex is generated, making an - * overriding implementation impossible to maintain. - */ +// This generates a private method since the predIndex is generated, making an +// overriding implementation impossible to maintain. +// +// We suppress UNSAFE_CALL as in "_localctx.something" _localctx is null, +// but we should be able to compile anyway. +// An alternative is to change RetValueRef to "_localctx!!.". RuleSempredFunction(r, actions) ::= << -private fun _sempred(_localctx: , predIndex: Int): Boolean { +@Suppress("UNSAFE_CALL") +private fun _sempred(_localctx: ?, predIndex: Int): Boolean { when (predIndex) { -> return ()}; separator="\n"> } @@ -412,11 +417,11 @@ LeftRecursiveRuleFunction(currentRule, args, code, locals, ruleCtx, altLabelCtxs }; separator="\n"> - }>public fun (): { - return (0}>) + }>public fun (): { + return (0}>) } -private fun (_p: Int}>): { +private fun (_p: Int}>): { var _parentctx = context var _parentState = state var _localctx = (context, _parentState}>) @@ -621,11 +626,11 @@ offsetShift(shiftAmount, offset) ::= <% // produces more efficient bytecode when bits.tokens contains at most two items bitsetInlineComparison(s, bits) ::= <% - == }; separator=" || "> + == Tokens.}; separator=" || "> %> cases(tokens) ::= << -}; separator=", "> -> >> +}; separator=", "> -> >> InvokeRule(r, argExprsChunks) ::= << this.state = @@ -640,10 +645,10 @@ _ctx = ( -_token = match() +_token = match(Tokens.) = _token}; separator="\n"> -match() +match(Tokens.) >> @@ -722,13 +727,13 @@ ActionText(t) ::= "" ActionTemplate(t) ::= "" ArgRef(a) ::= "_localctx." LocalRef(a) ::= "_localctx." -RetValueRef(a) ::= "_localctx." +RetValueRef(a) ::= "_localctx." QRetValueRef(a) ::= ".!!." /** How to translate $tokenLabel */ TokenRef(t) ::= "." LabelRef(t) ::= "." -ListLabelRef(t) ::= "." +ListLabelRef(t) ::= "." SetAttr(s, rhsChunks) ::= ". = " TokenLabelType() ::= "" @@ -744,13 +749,13 @@ TokenPropertyRef_int(t) ::= "(.?.text!!.toInt() ?: 0)" RulePropertyRef_start(r) ::= "(.?.start)" RulePropertyRef_stop(r) ::= "(.?.stop)" -RulePropertyRef_text(r) ::= "(.?.let { _input.getText(it.start!!, it.stop!!) })" +RulePropertyRef_text(r) ::= "(.?.let { _input.getText(it.start, it.stop) })" RulePropertyRef_ctx(r) ::= "." RulePropertyRef_parser(r) ::= "this" ThisRulePropertyRef_start(r) ::= "_localctx.start" ThisRulePropertyRef_stop(r) ::= "_localctx.stop" -ThisRulePropertyRef_text(r) ::= "_input.getText(_localctx.start!!, _input.LT(-1)!!)" +ThisRulePropertyRef_text(r) ::= "_input.getText(_localctx.start, _input.LT(-1))" ThisRulePropertyRef_ctx(r) ::= "_localctx" ThisRulePropertyRef_parser(r) ::= "this" @@ -801,11 +806,11 @@ StructDecl(struct, ctorAttrs, attrs, getters, dispatchMethods, interfaces, exten public open class : ParserRuleContext : { override val ruleIndex: Int = Rules. - }; separator="\n"> + }; separator="\n"> }; separator="\n"> public constructor(parent: ParserRuleContext?, invokingState: Int}>) : super(parent, invokingState) { - = }; separator="\n"> + = }; separator="\n"> } @@ -814,7 +819,7 @@ public open class : ) { super.copyFrom(ctx) - = ctx.}; separator="\n"> + = ctx.}; separator="\n"> } @@ -825,7 +830,7 @@ public open class : : Context { - }; separator="\n"> + }; separator="\n"> }; separator="\n"> public constructor(ctx: Context) { @@ -959,7 +964,7 @@ public open class (input: CharStream) : (i public object Modes { public const val DEFAULT_MODE: Int = 0 - : Int = }; separator="\n"> + : Int = }; separator="\n"> }