From b41a24f314816bf641dd3f1621b53031a3959f6d Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 30 Dec 2014 14:03:00 +0100 Subject: [PATCH 1/3] Handle function calls to integers in model lexer correctly closes #15877 --- src/grammar/RustLexer.g4 | 60 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/src/grammar/RustLexer.g4 b/src/grammar/RustLexer.g4 index 00af6d358e5..88de5db41fe 100644 --- a/src/grammar/RustLexer.g4 +++ b/src/grammar/RustLexer.g4 @@ -112,8 +112,64 @@ LIT_INTEGER ; LIT_FLOAT - : [0-9][0-9_]* ( '.' {_input.LA(1) != '.'}? - | ('.' [0-9][0-9_]*)? ([eE] [-+]? [0-9][0-9_]*)? SUFFIX?) + : [0-9][0-9_]* ('.' { + /* dot followed by another dot is a range, no float */ + _input.LA(1) != '.' && + /* dot followed by an identifier is an integer with a function call, no float */ + _input.LA(1) != '_' && + _input.LA(1) != 'a' && + _input.LA(1) != 'b' && + _input.LA(1) != 'c' && + _input.LA(1) != 'd' && + _input.LA(1) != 'e' && + _input.LA(1) != 'f' && + _input.LA(1) != 'g' && + _input.LA(1) != 'h' && + _input.LA(1) != 'i' && + _input.LA(1) != 'j' && + _input.LA(1) != 'k' && + _input.LA(1) != 'l' && + _input.LA(1) != 'm' && + _input.LA(1) != 'n' && + _input.LA(1) != 'o' && + _input.LA(1) != 'p' && + _input.LA(1) != 'q' && + _input.LA(1) != 'r' && + _input.LA(1) != 's' && + _input.LA(1) != 't' && + _input.LA(1) != 'u' && + _input.LA(1) != 'v' && + _input.LA(1) != 'w' && + _input.LA(1) != 'x' && + _input.LA(1) != 'y' && + _input.LA(1) != 'z' && + _input.LA(1) != 'A' && + _input.LA(1) != 'B' && + _input.LA(1) != 'C' && + _input.LA(1) != 'D' && + _input.LA(1) != 'E' && + _input.LA(1) != 'F' && + _input.LA(1) != 'G' && + _input.LA(1) != 'H' && + _input.LA(1) != 'I' && + _input.LA(1) != 'J' && + _input.LA(1) != 'K' && + _input.LA(1) != 'L' && + _input.LA(1) != 'M' && + _input.LA(1) != 'N' && + _input.LA(1) != 'O' && + _input.LA(1) != 'P' && + _input.LA(1) != 'Q' && + _input.LA(1) != 'R' && + _input.LA(1) != 'S' && + _input.LA(1) != 'T' && + _input.LA(1) != 'U' && + _input.LA(1) != 'V' && + _input.LA(1) != 'W' && + _input.LA(1) != 'X' && + _input.LA(1) != 'Y' && + _input.LA(1) != 'Z' + }? | ('.' [0-9][0-9_]*)? ([eE] [-+]? [0-9][0-9_]*)? SUFFIX?) ; LIT_STR From f066acf6450fcca60251eef6821cebd8f1ca9af3 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 30 Dec 2014 16:46:07 +0100 Subject: [PATCH 2/3] src/grammar/check.sh now prints test summary --- src/grammar/check.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/grammar/check.sh b/src/grammar/check.sh index f2836312437..cb269bbdb0a 100755 --- a/src/grammar/check.sh +++ b/src/grammar/check.sh @@ -11,6 +11,10 @@ if [ "${VERBOSE}" == "1" ]; then set -x fi +passed=0 +failed=0 +skipped=0 + check() { grep --silent "// ignore-lexer-test" $1; @@ -21,14 +25,27 @@ check() { # seem to have anny effect. if $3 RustLexer tokens -tokens < $1 | $4 $1 $5; then echo "pass: $1" + passed=`expr $passed + 1` else echo "fail: $1" + failed=`expr $failed + 1` fi else echo "skip: $1" + skipped=`expr $skipped + 1` fi } for file in $(find $1 -iname '*.rs' ! -path '*/test/compile-fail*'); do check $file $2 $3 $4 $5 done + +printf "\ntest result: " + +if [ $failed -eq 0 ]; then + printf "ok. $passed passed; $failed failed; $skipped skipped\n\n" +else + printf "failed. $passed passed; $failed failed; $skipped skipped\n\n" + exit 1 +fi + From 1e278c1cd1307d06d57c556676e3035516334d2d Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 30 Dec 2014 16:49:27 +0100 Subject: [PATCH 3/3] Update src/grammar/README.md --- src/grammar/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/grammar/README.md b/src/grammar/README.md index f5b872cdc7f..1f7923e1caf 100644 --- a/src/grammar/README.md +++ b/src/grammar/README.md @@ -1,7 +1,7 @@ Reference grammar. Uses [antlr4](http://www.antlr.org/) and a custom Rust tool to compare -ASTs/token streams generated. You can use the `check-syntax` make target to +ASTs/token streams generated. You can use the `check-lexer` make target to run all of the available tests. To use manually: @@ -12,7 +12,7 @@ javac *.java rustc -O verify.rs for file in ../*/**.rs; do echo $file; - grun RustLexer tokens -tokens < $file | ./verify $file || break + grun RustLexer tokens -tokens < $file | ./verify $file RustLexer.tokens || break done ```