[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[PATCH for Dlang support 1/2] d: change the return value of yylex from T
|
From: |
Adela Vais |
|
Subject: |
[PATCH for Dlang support 1/2] d: change the return value of yylex from TokenKind to YYParser.Symbol |
|
Date: |
Fri, 20 Nov 2020 16:40:54 +0200 |
The complete symbol approach was deemed to be the right approach for Dlang.
Now, the user can return from yylex() an instance of YYParser.Symbol structure,
which binds together the TokenKind, the semantic value and the location. Before,
the last two were reported separately to the parser.
Only the user API is changed, Bisons's internal structure is kept the same.
* data/skeletons/d.m4 (struct YYParser.Symbol): New.
* data/skeletons/lalr1.d: Change the return value.
* doc/bison.texi: Document it.
* examples/d/calc/calc.y, examples/d/simple/calc.y: Demonstrate it.
* tests/calc.at, tests/scanner.at: Test it.
---
data/skeletons/d.m4 | 35 +++++++++++++++++++++++++++++++++++
data/skeletons/lalr1.d | 19 +++++++++----------
doc/bison.texi | 6 +++---
examples/d/calc/calc.y | 20 ++++++++++----------
examples/d/simple/calc.y | 21 ++++++++++-----------
tests/calc.at | 30 +++++++++++++++---------------
tests/d.at | 8 ++++----
tests/scanner.at | 18 +++++++++---------
8 files changed, 95 insertions(+), 62 deletions(-)
diff --git a/data/skeletons/d.m4 b/data/skeletons/d.m4
index 7a501678..b84d4e94 100644
--- a/data/skeletons/d.m4
+++ b/data/skeletons/d.m4
@@ -446,3 +446,38 @@ m4_define([b4_var_decls],
], [$@])])
m4_define([b4_var_decl],
[ protected $1;])
+
+
+# b4_symbol_type_define
+# ---------------------
+# Define symbol_type, the external type for symbols used for symbol
+# constructors.
+m4_define([b4_symbol_type_define],
+[[
+ /**
+ * A complete symbol
+ */
+ struct Symbol
+ {
+ private SymbolKind kind;
+ private ]b4_yystype[ value;]b4_locations_if([[
+ private YYLocation location_;]])[
+ this(TokenKind token]b4_locations_if([[, YYLocation loc]])[)
+ {
+ kind = yytranslate_(token);]b4_locations_if([
+ location_ = loc;])[
+ }
+ static foreach (member; __traits(allMembers, YYSemanticType))
+ {
+ this(TokenKind token, typeof(mixin("YYSemanticType." ~ member))
val]b4_locations_if([[, YYLocation loc]])[)
+ {
+ kind = yytranslate_(token);
+ mixin("value." ~ member ~ " = val;");]b4_locations_if([
+ location_ = loc;])[
+ }
+ }
+ SymbolKind token() { return kind; }
+ ]b4_yystype[ semanticValue() { return value; }]b4_locations_if([[
+ YYLocation location() { return location_; }]])[
+ }
+]])
diff --git a/data/skeletons/lalr1.d b/data/skeletons/lalr1.d
index bf95e929..9fee542e 100644
--- a/data/skeletons/lalr1.d
+++ b/data/skeletons/lalr1.d
@@ -75,7 +75,7 @@ public interface Lexer
* to the next token and prepares to return the semantic value
* ]b4_locations_if([and beginning/ending positions ])[of the token.
* @@return the token identifier corresponding to the next token. */
- TokenKind yylex ();
+ ]b4_parser_class[.Symbol yylex ();
/**
* Entry point for error reporting. Emits an error
@@ -290,7 +290,7 @@ b4_user_union_members
yyDebugStream.writeln (s);
}
]])[
- private final TokenKind yylex () {
+ private final ]b4_parser_class[.Symbol yylex () {
return yylexer.yylex ();
}
@@ -411,7 +411,9 @@ b4_locations_if([, ref ]b4_location_type[ yylocationp])[)
yycdebugln (message);
}
}
-]])[
+]])
+b4_symbol_type_define
+[
/**
* Parse input from the scanner that was specified at object construction
* time. Return whether the end of the input was reached successfully.
@@ -493,13 +495,10 @@ m4_popdef([b4_at_dollar])])dnl
if (yychar == TokenKind.]b4_symbol(empty, id)[)
{]b4_parse_trace_if([[
yycdebugln ("Reading a token");]])[
- yychar = yylex ();]b4_locations_if([[
- static if (yy_location_is_class) {
- yylloc = new ]b4_location_type[(yylexer.startPos, yylexer.endPos);
- } else {
- yylloc = ]b4_location_type[(yylexer.startPos, yylexer.endPos);
- }]])
- yylval = yylexer.semanticVal;[
+ Symbol yysymbol = yylex();
+ yychar = yysymbol.token();
+ yylval = yysymbol.semanticValue();]b4_locations_if([[
+ yylloc = yysymbol.location();]])[
}
/* Convert token to internal form. */
diff --git a/doc/bison.texi b/doc/bison.texi
index 3ad56fdf..18b5dd2e 100644
--- a/doc/bison.texi
+++ b/doc/bison.texi
@@ -14011,9 +14011,9 @@ This method is defined by the user to emit an error
message. The first
parameter is omitted if location tracking is not active.
@end deftypemethod
-@deftypemethod {Lexer} {TokenKind} yylex()
-Return the next token. Its type is the return value, its semantic value and
-location are saved and returned by the their methods in the interface.
+@deftypemethod {Lexer} {YYParser.Symbol} yylex()
+Return the next token. The return value is of type YYParser.Symbol, which
+binds together the TokenKind, the semantic value and the location.
@end deftypemethod
@deftypemethod {Lexer} {YYPosition} getStartPos()
diff --git a/examples/d/calc/calc.y b/examples/d/calc/calc.y
index bee3fc94..3f41f048 100644
--- a/examples/d/calc/calc.y
+++ b/examples/d/calc/calc.y
@@ -114,7 +114,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_;
}
- TokenKind yylex()
+ Calc.Symbol yylex()
{
import std.uni : isWhite, isNumber;
@@ -127,7 +127,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
}
if (input.empty)
- return TokenKind.YYEOF;
+ return Calc.Symbol(TokenKind.YYEOF, new YYLocation(startPos, endPos));
// Numbers.
if (input.front.isNumber)
@@ -143,7 +143,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
}
start = end;
end.column += lenChars;
- return TokenKind.NUM;
+ return Calc.Symbol(TokenKind.NUM, semanticVal_.ival, new
YYLocation(startPos, endPos));
}
// Individual characters
@@ -153,17 +153,17 @@ if (isInputRange!R && is(ElementType!R : dchar))
end.column++;
switch (ch)
{
- case '+': return TokenKind.PLUS;
- case '-': return TokenKind.MINUS;
- case '*': return TokenKind.STAR;
- case '/': return TokenKind.SLASH;
- case '(': return TokenKind.LPAR;
- case ')': return TokenKind.RPAR;
+ case '+': return Calc.Symbol(TokenKind.PLUS, new YYLocation(startPos,
endPos));
+ case '-': return Calc.Symbol(TokenKind.MINUS, new YYLocation(startPos,
endPos));
+ case '*': return Calc.Symbol(TokenKind.STAR, new YYLocation(startPos,
endPos));
+ case '/': return Calc.Symbol(TokenKind.SLASH, new YYLocation(startPos,
endPos));
+ case '(': return Calc.Symbol(TokenKind.LPAR, new YYLocation(startPos,
endPos));
+ case ')': return Calc.Symbol(TokenKind.RPAR, new YYLocation(startPos,
endPos));
case '\n':
{
end.line++;
end.column = 1;
- return TokenKind.EOL;
+ return Calc.Symbol(TokenKind.EOL, new YYLocation(startPos, endPos));
}
default: assert(0);
}
diff --git a/examples/d/simple/calc.y b/examples/d/simple/calc.y
index 4cb07ca6..ff20673d 100644
--- a/examples/d/simple/calc.y
+++ b/examples/d/simple/calc.y
@@ -109,7 +109,7 @@ if (isInputRange!R && is(ElementType!R : dchar))
return semanticVal_;
}
- TokenKind yylex()
+ Calc.Symbol yylex()
{
import std.uni : isWhite, isNumber;
@@ -118,14 +118,13 @@ if (isInputRange!R && is(ElementType!R : dchar))
input.popFront;
if (input.empty)
- return TokenKind.YYEOF;
+ return Calc.Symbol(TokenKind.YYEOF);
// Numbers.
if (input.front.isNumber)
{
import std.conv : parse;
- semanticVal_.ival = input.parse!int;
- return TokenKind.NUM;
+ return Calc.Symbol(TokenKind.NUM, input.parse!int);
}
// Individual characters
@@ -133,13 +132,13 @@ if (isInputRange!R && is(ElementType!R : dchar))
input.popFront;
switch (ch)
{
- case '+': return TokenKind.PLUS;
- case '-': return TokenKind.MINUS;
- case '*': return TokenKind.STAR;
- case '/': return TokenKind.SLASH;
- case '(': return TokenKind.LPAR;
- case ')': return TokenKind.RPAR;
- case '\n': return TokenKind.EOL;
+ case '+': return Calc.Symbol(TokenKind.PLUS);
+ case '-': return Calc.Symbol(TokenKind.MINUS);
+ case '*': return Calc.Symbol(TokenKind.STAR);
+ case '/': return Calc.Symbol(TokenKind.SLASH);
+ case '(': return Calc.Symbol(TokenKind.LPAR);
+ case ')': return Calc.Symbol(TokenKind.RPAR);
+ case '\n': return Calc.Symbol(TokenKind.EOL);
default: assert(0);
}
}
diff --git a/tests/calc.at b/tests/calc.at
index 57dc2a14..d1a9d030 100644
--- a/tests/calc.at
+++ b/tests/calc.at
@@ -590,7 +590,7 @@ class CalcLexer(R) : Lexer
return res;
}
- TokenKind yylex ()
+ YYParser.Symbol yylex ()
{]AT_LOCATION_IF([[
location.begin = location.end;]])[
@@ -606,13 +606,13 @@ class CalcLexer(R) : Lexer
// EOF.
if (input.empty)
- return TokenKind.]AT_TOKEN_PREFIX[EOF;
+ return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOF]AT_LOCATION_IF([[,
new YYLocation(startPos, endPos)]])[);
// Numbers.
if (input.front.isNumber)
{
semanticVal_.ival = parseInt;
- return TokenKind.]AT_TOKEN_PREFIX[NUM;
+ return YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NUM,
semanticVal_.ival]AT_LOCATION_IF([[, new YYLocation(startPos, endPos)]])[);
}
// Individual characters
@@ -630,22 +630,22 @@ class CalcLexer(R) : Lexer
if (c == '#')
{
stderr.writeln (]AT_LOCATION_IF([location, ": ", ])["syntax error:
invalid character: '#'");
- return TokenKind.]AT_TOKEN_PREFIX[YYerror;
+ return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYerror]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
}
switch (c)
{
- case '+': return TokenKind.]AT_TOKEN_PREFIX[PLUS;
- case '-': return TokenKind.]AT_TOKEN_PREFIX[MINUS;
- case '*': return TokenKind.]AT_TOKEN_PREFIX[STAR;
- case '/': return TokenKind.]AT_TOKEN_PREFIX[SLASH;
- case '(': return TokenKind.]AT_TOKEN_PREFIX[LPAR;
- case ')': return TokenKind.]AT_TOKEN_PREFIX[RPAR;
- case '\n': return TokenKind.]AT_TOKEN_PREFIX[EOL;
- case '=': return TokenKind.]AT_TOKEN_PREFIX[EQUAL;
- case '^': return TokenKind.]AT_TOKEN_PREFIX[POW;
- case '!': return TokenKind.]AT_TOKEN_PREFIX[NOT;
- default: return TokenKind.]AT_TOKEN_PREFIX[YYUNDEF;
+ case '+': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[PLUS]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '-': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[MINUS]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '*': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[STAR]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '/': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[SLASH]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '(': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[LPAR]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case ')': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[RPAR]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '\n': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EOL]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '=': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[EQUAL]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '^': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[POW]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ case '!': return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[NOT]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
+ default: return
YYParser.Symbol(TokenKind.]AT_TOKEN_PREFIX[YYUNDEF]AT_LOCATION_IF([[, new
YYLocation(startPos, endPos)]])[);
}
}
}
diff --git a/tests/d.at b/tests/d.at
index 5ed45ee5..38268c4c 100644
--- a/tests/d.at
+++ b/tests/d.at
@@ -81,7 +81,7 @@ class CalcLexer(R) : Lexer
YYSemanticType semanticVal_;
YYSemanticType semanticVal() @property { return semanticVal_; }
- TokenKind yylex()
+ YYParser.Symbol yylex()
{
$2
}
@@ -143,16 +143,16 @@ AT_KEYWORDS([d])
AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw true
-%union { int ival; }], [return TokenKind.END;])
+%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])
AT_CHECK_D_MINIMAL_W_LEXER([
%define api.token.raw false
-%union { int ival; }], [return TokenKind.END;])
+%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 258,]])
AT_CHECK_D_MINIMAL_W_LEXER([
-%union { int ival; }], [return TokenKind.END;])
+%union { int ival; }], [return YYParser.Symbol(TokenKind.END);])
AT_CHECK_D_GREP([[ END = 3,]])
AT_CLEANUP
diff --git a/tests/scanner.at b/tests/scanner.at
index 1eb898bd..f19607ff 100644
--- a/tests/scanner.at
+++ b/tests/scanner.at
@@ -121,12 +121,12 @@ class YYLexer(R) : Lexer
return semanticVal_;
}
- TokenKind yylex ()
+ YYParser.Symbol yylex ()
{
import std.uni : isNumber;
// Handle EOF.
if (input.empty)
- return TokenKind.END;
+ return YYParser.Symbol(TokenKind.END);
auto c = input.front;
input.popFront;
@@ -136,13 +136,13 @@ class YYLexer(R) : Lexer
{
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
semanticVal_.val = c - '0';
- return TokenKind.NUM;
- case '+': return TokenKind.PLUS;
- case '-': return TokenKind.MINUS;
- case '*': return TokenKind.STAR;
- case '/': return TokenKind.SLASH;
- case '(': return TokenKind.LPAR;
- case ')': return TokenKind.RPAR;
+ return YYParser.Symbol(TokenKind.NUM, semanticVal_.val);
+ case '+': return YYParser.Symbol(TokenKind.PLUS);
+ case '-': return YYParser.Symbol(TokenKind.MINUS);
+ case '*': return YYParser.Symbol(TokenKind.STAR);
+ case '/': return YYParser.Symbol(TokenKind.SLASH);
+ case '(': return YYParser.Symbol(TokenKind.LPAR);
+ case ')': return YYParser.Symbol(TokenKind.RPAR);
default: assert(0);
}
}
--
2.17.1
- [PATCH for Dlang support 1/2] d: change the return value of yylex from TokenKind to YYParser.Symbol,
Adela Vais <=