    1   package org.codehaus.groovy.syntax.lexer;
    2
    3   import groovy.util.GroovyTestCase;
    4
    5   import org.codehaus.groovy.syntax.Token;
    6
    7   public class LexerTest extends GroovyTestCase {
    8
    9       public void testNothing()
    10      {
    11      }
    12
    13  /*
    14
    15      private Lexer lexer;
    16
    17      public void testEndOfStream() throws Exception {
    18          newLexer("");
    19
    20          assertEnd();
    21
    22          assertEnd();
    23
    24          assertEnd();
    25
    26          assertEnd();
    27      }
    28
    29      public void testSingleLineComment_Newline() throws Exception {
    30          newLexer("// I like cheese\ncheese");
    31
    32          assertNextToken(Token.NEWLINE, "<newline>");
    33          assertNextToken(Token.IDENTIFIER, "cheese");
    34
    35          assertEnd();
    36      }
    37
    38      public void testSingleLineComment_CarriageReturn() throws Exception {
    39          newLexer("// I like cheese\rcheese");
    40
    41          assertNextToken(Token.NEWLINE, "<newline>");
    42          assertNextToken(Token.IDENTIFIER, "cheese");
    43
    44          assertEnd();
    45      }
    46
    47      public void testSingleLineComment_CarriageReturn_Newline() throws Exception {
    48          newLexer("// I like cheese\r\ncheese");
    49
    50          assertNextToken(Token.NEWLINE, "<newline>");
    51          assertNextToken(Token.IDENTIFIER, "cheese");
    52
    53          assertEnd();
    54      }
    55
    56      public void testSingleLineHashComment_CarriageReturn() throws Exception {
    57          newLexer("# I like cheese\rcheese");
    58
    59          assertNextToken(Token.NEWLINE, "<newline>");
    60          assertNextToken(Token.IDENTIFIER, "cheese");
    61
    62          assertEnd();
    63      }
    64
    65      public void testMultilineComment_MiddleOfLine() throws Exception {
    66  */ //        newLexer("cheese /* is */ toasty");
    67  /*
    68
    69          assertNextToken(Token.IDENTIFIER, "cheese");
    70
    71          assertNextToken(Token.IDENTIFIER, "toasty");
    72
    73          assertEnd();
    74      }
    75
    76      public void testMultilineComment_SpanningLines() throws Exception {
    77  */ //        newLexer("cheese /* is \n really */ toasty");
    78  /*
    79
    80          assertNextToken(Token.IDENTIFIER, "cheese");
    81
    82          assertNextToken(Token.IDENTIFIER, "toasty");
    83
    84          assertEnd();
    85      }
    86
    87      public void testMultilineComment_EmbeddedStarts() throws Exception {
    88  */ //        newLexer("cheese /* * * * / * / */ toasty");
    89  /*
    90
    91          assertNextToken(Token.IDENTIFIER, "cheese");
    92
    93          assertNextToken(Token.IDENTIFIER, "toasty");
    94
    95          assertEnd();
    96      }
    97
    98      public void testIgnoredWhitespace() throws Exception {
    99          newLexer(" \r \n \r\n \n\r    \t   \t");
    100
    101         assertNextToken(Token.NEWLINE, "<newline>");
    102         assertNextToken(Token.NEWLINE, "<newline>");
    103         assertNextToken(Token.NEWLINE, "<newline>");
    104
    105         assertEnd();
    106     }
    107
    108     public void testLeftCurlyBrace() throws Exception {
    109         assertSimple("{", Token.LEFT_CURLY_BRACE);
    110     }
    111
    112     public void testRightCurlyBrace() throws Exception {
    113         assertSimple("}", Token.RIGHT_CURLY_BRACE);
    114     }
    115
    116     public void testLeftSquareBracket() throws Exception {
    117         assertSimple("[", Token.LEFT_SQUARE_BRACKET);
    118     }
    119
    120     public void testRightSquareBracket() throws Exception {
    121         assertSimple("]", Token.RIGHT_SQUARE_BRACKET);
    122     }
    123
    124     public void testLeftParenthesis() throws Exception {
    125         assertSimple("(", Token.LEFT_PARENTHESIS);
    126     }
    127
    128     public void testRightParenthesis() throws Exception {
    129         assertSimple(")", Token.RIGHT_PARENTHESIS);
    130     }
    131
    132     public void testDot() throws Exception {
    133         assertSimple(".", Token.DOT);
    134     }
    135
    136     public void testDotDot() throws Exception {
    137         assertSimple("..", Token.DOT_DOT);
    138     }
    139
    140     public void testNot() throws Exception {
    141         assertSimple("!", Token.NOT);
    142     }
    143
    144     public void testCompareNotEqual() throws Exception {
    145         assertSimple("!=", Token.COMPARE_NOT_EQUAL);
    146     }
    147
    148     public void testEqual() throws Exception {
    149         assertSimple("=", Token.EQUAL);
    150     }
    151
    152     public void testCompareEqual() throws Exception {
    153         assertSimple("==", Token.COMPARE_EQUAL);
    154     }
    155
    156     public void testCompareIdentical() throws Exception {
    157         assertSimple("===", Token.COMPARE_IDENTICAL);
    158     }
    159
    160     public void testCompareLessThan() throws Exception {
    161         assertSimple("<", Token.COMPARE_LESS_THAN);
    162     }
    163
    164     public void testCompareLessThanEqual() throws Exception {
    165         assertSimple("<=", Token.COMPARE_LESS_THAN_EQUAL);
    166     }
    167
    168     public void testCompareGreaterThan() throws Exception {
    169         assertSimple(">", Token.COMPARE_GREATER_THAN);
    170     }
    171
    172     public void testCompareGreaterThanEqual() throws Exception {
    173         assertSimple(">=", Token.COMPARE_GREATER_THAN_EQUAL);
    174     }
    175
    176     public void testCompareTo() throws Exception {
    177         assertSimple("<=>", Token.COMPARE_TO);
    178     }
    179
    180     public void testNavigate() throws Exception {
    181         assertSimple("->", Token.NAVIGATE);
    182     }
    183
    184     public void testLogicalOr() throws Exception {
    185         assertSimple("||", Token.LOGICAL_OR);
    186     }
    187
    188     public void testPipe() throws Exception {
    189         assertSimple("|", Token.PIPE);
    190     }
    191
    192     public void testLogicalAnd() throws Exception {
    193         assertSimple("&&", Token.LOGICAL_AND);
    194     }
    195
    196     public void testAmpersand_UnexpectedCharacter() throws Exception {
    197         newLexer("&a");
    198
    199         char[] expected = assertUnexpectedCharacter('a', 1, 2);
    200
    201         assertLength(1, expected);
    202
    203         assertContains('&', expected);
    204     }
    205
    206     public void testPlus() throws Exception {
    207         assertSimple("+", Token.PLUS);
    208     }
    209
    210     public void testPlusPlus() throws Exception {
    211         assertSimple("++", Token.PLUS_PLUS);
    212     }
    213
    214     public void testPlusEqual() throws Exception {
    215         assertSimple("+=", Token.PLUS_EQUAL);
    216     }
    217
    218     public void testMinus() throws Exception {
    219         assertSimple("-", Token.MINUS);
    220     }
    221
    222     public void testMinusMinus() throws Exception {
    223         assertSimple("--", Token.MINUS_MINUS);
    224     }
    225
    226     public void testMinusEqual() throws Exception {
    227         assertSimple("-=", Token.MINUS_EQUAL);
    228     }
    229
    230     public void testDivide() throws Exception {
    231         assertSimple("/", Token.DIVIDE);
    232     }
    233
    234     public void testDivideEqual() throws Exception {
    235         assertSimple("/=", Token.DIVIDE_EQUAL);
    236     }
    237
    238     public void testMod() throws Exception {
    239         assertSimple("%", Token.MOD);
    240     }
    241
    242     public void testModEqual() throws Exception {
    243         assertSimple("%=", Token.MOD_EQUAL);
    244     }
    245
    246     public void testMultiply() throws Exception {
    247         assertSimple("*", Token.MULTIPLY);
    248     }
    249
    250     public void testMultiplyEqual() throws Exception {
    251         assertSimple("*=", Token.MULTIPLY_EQUAL);
    252     }
    253
    254     public void testColon() throws Exception {
    255         assertSimple(":", Token.COLON);
    256     }
    257
    258     public void testSemicolon() throws Exception {
    259         assertSimple(";", Token.SEMICOLON);
    260     }
    261
    262     public void testQuestion() throws Exception {
    263         assertSimple("?", Token.QUESTION);
    264     }
    265
    266     public void testDoubleQuoteString_Simple() throws Exception {
    267         newLexer("\"cheese\"");
    268
    269         assertNextToken(Token.DOUBLE_QUOTE_STRING, "cheese");
    270
    271         assertEnd();
    272     }
    273
    274     public void testDoubleQuoteString_EscapedDollar() throws Exception {
    275         newLexer("\"foo $${bar}\"");
    276
    277         assertNextToken(Token.DOUBLE_QUOTE_STRING, "foo $${bar}");
    278
    279         assertEnd();
    280     }
    281
    282     public void testDoubleQuoteString_EscapedTab() throws Exception {
    283         newLexer("\"che//tese\"");
    284
    285         assertNextToken(Token.DOUBLE_QUOTE_STRING, "che\tese");
    286
    287         assertEnd();
    288     }
    289
    290     public void testDoubleQuoteString_EscapedNewline() throws Exception {
    291         newLexer("\"che//nese\"");
    292
    293         assertNextToken(Token.DOUBLE_QUOTE_STRING, "che\nese");
    294
    295         assertEnd();
    296     }
    297
    298     public void testDoubleQuoteString_EscapedCarriageReturn() throws Exception {
    299         newLexer("\"che//rese\"");
    300
    301         assertNextToken(Token.DOUBLE_QUOTE_STRING, "che\rese");
    302
    303         assertEnd();
    304     }
    305
    306     public void testDoubleQuoteString_EscapedOther() throws Exception {
    307         newLexer("\"che//bese\"");
    308
    309         assertNextToken(Token.DOUBLE_QUOTE_STRING, "chebese");
    310
    311         assertEnd();
    312     }
    313
    314     public void testSingleQuoteString_Simple() throws Exception {
    315         newLexer("'cheese'");
    316
    317         assertNextToken(Token.SINGLE_QUOTE_STRING, "cheese");
    318
    319         assertEnd();
    320     }
    321
    322     public void testSingleQuoteString_EscapedTab() throws Exception {
    323         newLexer("'che//tese'");
    324
    325         assertNextToken(Token.SINGLE_QUOTE_STRING, "che\tese");
    326
    327         assertEnd();
    328     }
    329
    330     public void testSingleQuoteString_EscapedNewline() throws Exception {
    331         newLexer("'che//nese'");
    332
    333         assertNextToken(Token.SINGLE_QUOTE_STRING, "che\nese");
    334
    335         assertEnd();
    336     }
    337
    338     public void testSingleQuoteString_EscapedCarriageReturn() throws Exception {
    339         newLexer("'che//rese'");
    340
    341         assertNextToken(Token.SINGLE_QUOTE_STRING, "che\rese");
    342
    343         assertEnd();
    344     }
    345
    346     public void testSingleQuoteString_EscapedOther() throws Exception {
    347         newLexer("'che//bese'");
    348
    349         assertNextToken(Token.SINGLE_QUOTE_STRING, "chebese");
    350
    351         assertEnd();
    352     }
    353
    354     public void testUnterminatedStringLiteral_DoubleQuote_Newline() throws Exception {
    355         newLexer("\"cheese\n");
    356
    357         try {
    358             nextToken();
    359             fail("should have thrown UnterminatedStringLiteralException");
    360         }
    361         catch (UnterminatedStringLiteralException e) {
    362             // expected and correct
    363         }
    364     }
    365
    366     public void testUnterminatedStringLiteral_DoubleQuote_CarriageReturn() throws Exception {
    367         newLexer("\"cheese\r");
    368
    369         try {
    370             nextToken();
    371             fail("should have thrown UnterminatedStringLiteralException");
    372         }
    373         catch (UnterminatedStringLiteralException e) {
    374             // expected and correct
    375         }
    376     }
    377
    378     public void testUnterminatedStringLiteral_DoubleQuote_EndOfStream() throws Exception {
    379         newLexer("\"cheese");
    380
    381         try {
    382             nextToken();
    383             fail("should have thrown UnterminatedStringLiteralException");
    384         }
    385         catch (UnterminatedStringLiteralException e) {
    386             // expected and correct
    387         }
    388     }
    389
    390     public void testUnterminatedStringLiteral_SingleQuote_Newline() throws Exception {
    391         newLexer("'cheese\n'");
    392
    393         try {
    394             nextToken();
    395             fail("should have thrown UnterminatedStringLiteralException");
    396         }
    397         catch (UnterminatedStringLiteralException e) {
    398             // expected and correct
    399         }
    400     }
    401
    402     public void testUnterminatedStringLiteral_SingleQuote_CarriageReturn() throws Exception {
    403         newLexer("'cheese\r'");
    404
    405         try {
    406             nextToken();
    407             fail("should have thrown UnterminatedStringLiteralException");
    408         }
    409         catch (UnterminatedStringLiteralException e) {
    410             // expected and correct
    411         }
    412     }
    413
    414     public void testUnterminatedStringLiteral_SingleQuote_EndOfStream() throws Exception {
    415         newLexer("'cheese");
    416
    417         try {
    418             nextToken();
    419             fail("should have thrown UnterminatedStringLiteralException");
    420         }
    421         catch (UnterminatedStringLiteralException e) {
    422             // expected and correct
    423         }
    424     }
    425
    426     public void testIdentifier() throws Exception {
    427         assertSimple("cheese", Token.IDENTIFIER);
    428     }
    429
    430     public void testNumber_Integer() throws Exception {
    431         assertSimple("42", Token.INTEGER_NUMBER);
    432     }
    433
    434     public void testNumber_FloatingPoint() throws Exception {
    435         assertSimple("42.84", Token.FLOAT_NUMBER);
    436     }
    437
    438     public void testNumber_IntegerCall() throws Exception {
    439         newLexer("42.cheese");
    440
    441         assertNextToken(Token.INTEGER_NUMBER, "42");
    442         assertNextToken(Token.DOT, ".");
    443         assertNextToken(Token.IDENTIFIER, "cheese");
    444     }
    445
    446     public void testNumber_FloatCall() throws Exception {
    447         newLexer("42.0.cheese");
    448
    449         assertNextToken(Token.FLOAT_NUMBER, "42.0");
    450         assertNextToken(Token.DOT, ".");
    451         assertNextToken(Token.IDENTIFIER, "cheese");
    452     }
    453
    454 */    /*
    455     public void testNumber_UnexpectedCharacter()
    456         throws Exception
    457     {
    458         newLexer( "4.0cheese" );
    459
    460         char[] expected = assertUnexpectedCharacter( 'c',
    461                                                      1,
    462                                                      4 );
    463
    464         assertLength( 10,
    465                       expected );
    466
    467         assertContains( '0',
    468                         expected );
    469         assertContains( '1',
    470                         expected );
    471         assertContains( '2',
    472                         expected );
    473         assertContains( '3',
    474                         expected );
    475         assertContains( '4',
    476                         expected );
    477         assertContains( '5',
    478                         expected );
    479         assertContains( '6',
    480                         expected );
    481         assertContains( '7',
    482                         expected );
    483         assertContains( '8',
    484                         expected );
    485         assertContains( '9',
    486                         expected );
    487     }
    488     */
    489 /*
    490
    491     // ----------------------------------------------------------------------
    492     // ----------------------------------------------------------------------
    493
    494     public void testKeyword_Abstract() throws Exception {
    495         assertSimple("abstract", Token.KEYWORD_ABSTRACT);
    496     }
    497
    498     public void testKeyword_As() throws Exception {
    499         assertSimple("as", Token.KEYWORD_AS);
    500     }
    501
    502     public void testKeyword_Break() throws Exception {
    503         assertSimple("break", Token.KEYWORD_BREAK);
    504     }
    505
    506     public void testKeyword_Case() throws Exception {
    507         assertSimple("case", Token.KEYWORD_CASE);
    508     }
    509
    510     public void testKeyword_Catch() throws Exception {
    511         assertSimple("catch", Token.KEYWORD_CATCH);
    512     }
    513
    514     public void testKeyword_Class() throws Exception {
    515         assertSimple("class", Token.KEYWORD_CLASS);
    516     }
    517
    518     public void testKeyword_Const() throws Exception {
    519         assertSimple("const", Token.KEYWORD_CONST);
    520     }
    521
    522     public void testKeyword_Continue() throws Exception {
    523         assertSimple("continue", Token.KEYWORD_CONTINUE);
    524     }
    525
    526     public void testKeyword_Default() throws Exception {
    527         assertSimple("default", Token.KEYWORD_DEFAULT);
    528     }
    529
    530     public void testKeyword_Do() throws Exception {
    531         assertSimple("do", Token.KEYWORD_DO);
    532     }
    533
    534     public void testKeyword_Else() throws Exception {
    535         assertSimple("else", Token.KEYWORD_ELSE);
    536     }
    537
    538     public void testKeyword_Extends() throws Exception {
    539         assertSimple("extends", Token.KEYWORD_EXTENDS);
    540     }
    541
    542     public void testKeyword_Final() throws Exception {
    543         assertSimple("final", Token.KEYWORD_FINAL);
    544     }
    545
    546     public void testKeyword_Finally() throws Exception {
    547         assertSimple("finally", Token.KEYWORD_FINALLY);
    548     }
    549
    550     public void testKeyword_For() throws Exception {
    551         assertSimple("for", Token.KEYWORD_FOR);
    552     }
    553
    554     public void testKeyword_Goto() throws Exception {
    555         assertSimple("goto", Token.KEYWORD_GOTO);
    556     }
    557
    558     public void testKeyword_If() throws Exception {
    559         assertSimple("if", Token.KEYWORD_IF);
    560     }
    561
    562     public void testKeyword_Implements() throws Exception {
    563         assertSimple("implements", Token.KEYWORD_IMPLEMENTS);
    564     }
    565
    566     public void testKeyword_Import() throws Exception {
    567         assertSimple("import", Token.KEYWORD_IMPORT);
    568     }
    569
    570     public void testKeyword_Instanceof() throws Exception {
    571         assertSimple("instanceof", Token.KEYWORD_INSTANCEOF);
    572     }
    573
    574     public void testKeyword_Interface() throws Exception {
    575         assertSimple("interface", Token.KEYWORD_INTERFACE);
    576     }
    577
    578     public void testKeyword_Native() throws Exception {
    579         assertSimple("native", Token.KEYWORD_NATIVE);
    580     }
    581
    582     public void testKeyword_New() throws Exception {
    583         assertSimple("new", Token.KEYWORD_NEW);
    584     }
    585
    586     public void testKeyword_Package() throws Exception {
    587         assertSimple("package", Token.KEYWORD_PACKAGE);
    588     }
    589
    590     public void testKeyword_Private() throws Exception {
    591         assertSimple("private", Token.KEYWORD_PRIVATE);
    592     }
    593
    594     public void testKeyword_Property() throws Exception {
    595         assertSimple("property", Token.KEYWORD_PROPERTY);
    596     }
    597
    598     public void testKeyword_Protected() throws Exception {
    599         assertSimple("protected", Token.KEYWORD_PROTECTED);
    600     }
    601
    602     public void testKeyword_Public() throws Exception {
    603         assertSimple("public", Token.KEYWORD_PUBLIC);
    604     }
    605
    606     public void testKeyword_Return() throws Exception {
    607         assertSimple("return", Token.KEYWORD_RETURN);
    608     }
    609
    610     public void testKeyword_Static() throws Exception {
    611         assertSimple("static", Token.KEYWORD_STATIC);
    612     }
    613
    614     public void testKeyword_Super() throws Exception {
    615         assertSimple("super", Token.KEYWORD_SUPER);
    616     }
    617
    618     public void testKeyword_Switch() throws Exception {
    619         assertSimple("switch", Token.KEYWORD_SWITCH);
    620     }
    621
    622     public void testKeyword_Synchronized() throws Exception {
    623         assertSimple("synchronized", Token.KEYWORD_SYNCHRONIZED);
    624     }
    625
    626     public void testKeyword_This() throws Exception {
    627         assertSimple("this", Token.KEYWORD_THIS);
    628     }
    629
    630     public void testKeyword_Throw() throws Exception {
    631         assertSimple("throw", Token.KEYWORD_THROW);
    632     }
    633
    634     public void testKeyword_Throws() throws Exception {
    635         assertSimple("throws", Token.KEYWORD_THROWS);
    636     }
    637
    638     public void testKeyword_Try() throws Exception {
    639         assertSimple("try", Token.KEYWORD_TRY);
    640     }
    641
    642     public void testKeyword_While() throws Exception {
    643         assertSimple("while", Token.KEYWORD_WHILE);
    644     }
    645
    646     public void testUnexpecteCharacterException() throws Exception {
    647         newLexer("±");
    648
    649         try {
    650             nextToken();
    651             fail("should have thrown UnexpectedCharacterException");
    652         }
    653         catch (UnexpectedCharacterException e) {
    654             // expected and correct
    655             assertEquals('±', e.getCharacter());
    656         }
    657     }
    658
    659     // ----------------------------------------------------------------------
    660     // ----------------------------------------------------------------------
    661
    662     protected void assertSimple(String text, int type) throws Exception {
    663         newLexer(text);
    664
    665         assertNextToken(type, text);
    666
    667         assertEnd();
    668     }
    669
    670     protected void assertNextToken(int type, String text) throws Exception {
    671         Token token = this.lexer.nextToken();
    672
    673         assertNotNull(token);
    674
    675         assertEquals(type, token.getType());
    676
    677         //System.out.println("Expected: " + text + " but got: " + token.getText());
    678
    679         assertEquals(text, token.getText());
    680     }
    681
    682     protected void nextToken() throws Exception {
    683         this.lexer.nextToken();
    684     }
    685
    686     protected char[] assertUnexpectedCharacter(char c, int line, int column) throws Exception {
    687         try {
    688             this.lexer.nextToken();
    689             fail("should have thrown UnexpectedCharacterException");
    690         }
    691         catch (UnexpectedCharacterException e) {
    692             // expected and correct
    693             assertEquals(c, e.getCharacter());
    694
    695             assertEquals(line, e.getLine());
    696
    697             assertEquals(column, e.getStartColumn());
    698
    699             return e.getExpected();
    700         }
    701
    702         return new char[] {
    703         };
    704     }
    705
    706     protected void assertEnd() throws Exception {
    707         assertNull(this.lexer.nextToken());
    708     }
    709
    710     protected void newLexer(String text) {
    711         StringCharStream in = new StringCharStream(text);
    712
    713         this.lexer = new Lexer(in);
    714     }
    715 */
    716 }