sublexer_impl -- unsyntaxの判断条件が誤っており、正しい結果が得られなかったのを修正

                 NestedCommentLexerのコンストラクタが誤っていたのを修正
lexer_test -- lexer_test2,lexer_test3を追加。

diff --git a/lexeme_impl.cpp b/lexeme_impl.cpp
index 058b473..57f76d2 100644 (file)
--- a/lexeme_impl.cpp
+++ b/lexeme_impl.cpp
@@ -256,7 +256,7 @@ int Syntax::getID() const
 smart_ptr<utakata::utf8_string::UTF8String> Syntax::toString() const
 {
     smart_ptr<utakata::utf8_string::UTF8String> p(new utf8_string::UTF8String);
-    *p += std::string("#`");
+    *p += std::string("#'");
     return p;
 }
 
@@ -272,7 +272,7 @@ int QuasiSyntax::getID() const
 smart_ptr<utakata::utf8_string::UTF8String> QuasiSyntax::toString() const
 {
     smart_ptr<utakata::utf8_string::UTF8String> p(new utf8_string::UTF8String);
-    *p += std::string("#'");
+    *p += std::string("#`");
     return p;
 }
 

diff --git a/sublexer_impl.cpp b/sublexer_impl.cpp
index 3321669..166454d 100755 (executable)
--- a/sublexer_impl.cpp
+++ b/sublexer_impl.cpp
@@ -141,10 +141,12 @@ smart_ptr<lexeme::ILexeme> sublexer::FirstLexer::lex_(const utakata::utf8_string
         UTF8Char ch = stream->peek();
         if (ch.toUTF16Code() == '\'')
         {
+            stream->read();
             return lexeme::makeSyntax();
         }
-        else if ('`')
+        else if (ch.toUTF16Code() == '`')
         {
+            stream->read();
             return lexeme::makeQuasiSyntax();
         }
         else if (ch.toUTF16Code() == ',')
@@ -167,7 +169,7 @@ smart_ptr<lexeme::ILexeme> sublexer::FirstLexer::lex_(const utakata::utf8_string
             // ネストコメントの開始という判断がなされ、ネストコメント
             // 用の字句解析に移る。
             stream->read();
-            next.add(new sublexer::NestedCommentLexer(str + ch));
+            next.add(new sublexer::NestedCommentLexer());
         }
         else if (ch.toUTF16Code() == 'e' || ch.toUTF16Code() == 'E' ||
                  ch.toUTF16Code() == 'i' || ch.toUTF16Code() == 'I' ||

diff --git a/sublexer_impl.h b/sublexer_impl.h
index fd4e59e..0979eec 100644 (file)
--- a/sublexer_impl.h
+++ b/sublexer_impl.h
@@ -81,7 +81,7 @@ namespace utakata {
             // を保存する必要すらない。
         public:
 
-            NestedCommentLexer(const utakata::utf8_string::UTF8String& str);
+            NestedCommentLexer(){}
             virtual ~NestedCommentLexer(){}
 
             smart_ptr<lexeme::ILexeme> lex(smart_ptr<utakata::utf8::UTF8InputStream> stream,

diff --git a/test/lexer_test.cpp b/test/lexer_test.cpp
index 06aae03..47d100e 100755 (executable)
--- a/test/lexer_test.cpp
+++ b/test/lexer_test.cpp
@@ -19,7 +19,7 @@ bool lexer_test(smart_ptr<simpletest::SimpleTestAsserter> asserter)
 {
     std::stringstream ss;
     ss << "==========" << endl;
-    ss << "(hoge 12 \"hoge\")" << endl;
+    ss << "(hoge 12 \"hoge\" )" << endl;
 
     textarrayformat::TextArrayReader reader(ss);
     smart_ptr<istream> formats(new stringstream(reader.get()));
@@ -54,12 +54,132 @@ bool lexer_test(smart_ptr<simpletest::SimpleTestAsserter> asserter)
     return asserter->isOk();
 }
 
+bool lexer_test2(smart_ptr<simpletest::SimpleTestAsserter> asserter)
+{
+    std::stringstream ss;
+    ss << "==========" << endl;
+    ss << ",@(hoge 12 `(hoge) 'huga)" << endl;
+
+    textarrayformat::TextArrayReader reader(ss);
+    smart_ptr<istream> formats(new stringstream(reader.get()));
+    smart_ptr<utakata::utf8::UTF8InputStream> st;
+    st.add(new utakata::utf8::UTF8InputStream(formats));
+    
+    utakata::lexer::Lexer lexer;
+
+    smart_ptr<lexeme::ILexeme> m(lexer.lex(st));
+    asserter->check(m->toString()->toStr(), ",@");
+
+    lexeme::IDProxy idp;
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->openParenthesis, "open paren");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->identifier, "identifier");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->number, "number");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->backquote, "back quote");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->openParenthesis, "open paren2");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->identifier, "identifier2");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->closeParenthesis, "close");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->quote, "quote");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->identifier, "identifier2");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->closeParenthesis, "close");
+
+    // ここで終了しているはず。EOFの状態から実行されると、EOFのIDを持った
+    // lexemeが返される。
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->eof, "eof");
+    
+    return asserter->isOk();
+}
+
+bool lexer_test3(smart_ptr<simpletest::SimpleTestAsserter> asserter)
+{
+    std::stringstream ss;
+    ss << "==========" << endl;
+    ss << "#'(hoge #,12 #`(hoge) 'huga) ; comment" << endl;
+    ss << "hoge" << endl;
+
+    textarrayformat::TextArrayReader reader(ss);
+    smart_ptr<istream> formats(new stringstream(reader.get()));
+    smart_ptr<utakata::utf8::UTF8InputStream> st;
+    st.add(new utakata::utf8::UTF8InputStream(formats));
+    
+    utakata::lexer::Lexer lexer;
+
+    smart_ptr<lexeme::ILexeme> m(lexer.lex(st));
+    asserter->check(m->toString()->toStr(), "#'");
+
+    lexeme::IDProxy idp;
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->openParenthesis, "open paren");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->identifier, "identifier");
+
+    m = lexer.lex(st);
+    asserter->check(m->toString()->toStr(), "#,");
+    asserter->check(m->getID(), idp->unsyntax, "unsyntax");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->number, "number");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->quasiSyntax, "quasi syntax");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->openParenthesis, "open paren2");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->identifier, "identifier2");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->closeParenthesis, "close");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->quote, "quote");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->identifier, "identifier2");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->closeParenthesis, "close");
+
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->identifier, "identifier3");
+
+
+    // ここで終了しているはず。EOFの状態から実行されると、EOFのIDを持った
+    // lexemeが返される。
+    m = lexer.lex(st);
+    asserter->check(m->getID(), idp->eof, "eof");
+    
+    return asserter->isOk();
+}
+
+
 int main(int argc, char *argv[])
 {
     simpletest::SimpleTestSuite suite("main lexer test");
     suite.addTester(sfcr::screate(lexer_test, suite.getAsserter()));
-//     suite.addTester(sfcr::screate(utf8_string_test, suite.getAsserter()));
-//     suite.addTester(sfcr::screate(utf8_string_util_test, suite.getAsserter()));
+    suite.addTester(sfcr::screate(lexer_test2, suite.getAsserter()));
+    suite.addTester(sfcr::screate(lexer_test3, suite.getAsserter()));
     suite.run();
     return 0;
 }